Struct polars_lazy::frame::LazyFrame

pub struct LazyFrame { /* fields omitted */ }

Lazy abstraction over an eager DataFrame. It really is an abstraction over a logical plan. The methods of this struct will incrementally modify a logical plan until output is requested (via collect)

Implementations

impl LazyFrame

pub fn new_from_parquet(
    path: String,
    stop_after_n_rows: Option<usize>,
    cache: bool
) -> Self

Create a LazyFrame directly from a parquet scan.

pub fn to_dot(&self, optimized: bool) -> Result<String>

Get a dot language representation of the LogicalPlan.

pub fn with_projection_pushdown(self, toggle: bool) -> Self

Toggle projection pushdown optimization.

pub fn with_predicate_pushdown(self, toggle: bool) -> Self

Toggle predicate pushdown optimization.

pub fn with_type_coercion(self, toggle: bool) -> Self

Toggle type coercion optimization.

pub fn with_simplify_expr(self, toggle: bool) -> Self

Toggle expression simplification optimization on or off

pub fn with_aggregate_pushdown(self, toggle: bool) -> Self

Toggle aggregate pushdown.

pub fn with_string_cache(self, toggle: bool) -> Self

Toggle global string cache.

pub fn with_join_pruning(self, toggle: bool) -> Self

Toggle join pruning optimization

pub fn describe_plan(&self) -> String

Describe the logical plan.

pub fn describe_optimized_plan(&self) -> Result<String>

Describe the optimized logical plan.

pub fn sort(self, by_column: &str, reverse: bool) -> Self

Add a sort operation to the logical plan.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

/// Sort DataFrame by 'sepal.width' column
fn example(df: DataFrame) -> LazyFrame {
      df.lazy()
        .sort("sepal.width", false)
}

pub fn sort_by_exprs(self, by_exprs: Vec<Expr>, reverse: Vec<bool>) -> Self

Add a sort operation to the logical plan.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

/// Sort DataFrame by 'sepal.width' column
fn example(df: DataFrame) -> LazyFrame {
      df.lazy()
        .sort_by_exprs(vec![col("sepal.width")], vec![false])
}

pub fn reverse(self) -> Self

Reverse the DataFrame

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

fn example(df: DataFrame) -> LazyFrame {
      df.lazy()
        .reverse()
}

pub fn with_column_renamed(self, existing_name: &str, new_name: &str) -> Self

Rename a column in the DataFrame

pub fn shift(self, periods: i64) -> Self

Shift the values by a given period and fill the parts that will be empty due to this operation with Nones.

See the method on Series for more info on the shift operation.

pub fn shift_and_fill(self, periods: i64, fill_value: Expr) -> Self

Shift the values by a given period and fill the parts that will be empty due to this operation with the result of the fill_value expression.

See the method on Series for more info on the shift operation.

pub fn fill_none(self, fill_value: Expr) -> LazyFrame

Fill none values in the DataFrame

pub fn cache(self) -> Self

Caches the result into a new LazyFrame. This should be used to prevent computations running multiple times

pub fn fetch(self, n_rows: usize) -> Result<DataFrame>

Fetch is like a collect operation, but it overwrites the number of rows read by every scan operation. This is a utility that helps debug a query on a smaller number of rows.

Note that the fetch does not guarantee the final number of rows in the DataFrame. Filter, join operations and a lower number of rows available in the scanned file influence the final number of rows.

pub fn optimize(
    self,
    lp_arena: &mut Arena<ALogicalPlan>,
    expr_arena: &mut Arena<AExpr>
) -> Result<Node>

pub fn collect(self) -> Result<DataFrame>

Execute all the lazy operations and collect them into a DataFrame. Before execution the query is being optimized.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

fn example(df: DataFrame) -> Result<DataFrame> {
      df.lazy()
        .groupby(vec![col("foo")])
        .agg(vec!(col("bar").sum(),
                  col("ham").mean().alias("avg_ham")))
        .collect()
}

pub fn filter(self, predicate: Expr) -> Self

Filter by some predicate expression.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

fn example(df: DataFrame) -> LazyFrame {
      df.lazy()
        .filter(col("sepal.width").is_not_null())
        .select(&[col("sepal.width"), col("sepal.length")])
}

pub fn select<E: AsRef<[Expr]>>(self, exprs: E) -> Self

Select (and rename) columns from the query.

Columns can be selected with col; If you want to select all columns use col("*").

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

/// This function selects column "foo" and column "bar".
/// Column "bar" is renamed to "ham".
fn example(df: DataFrame) -> LazyFrame {
      df.lazy()
        .select(&[col("foo"),
                  col("bar").alias("ham")])
}

/// This function selects all columns except "foo"
fn exclude_a_column(df: DataFrame) -> LazyFrame {
      df.lazy()
        .select(&[col("*"),
                  except("foo")])
}

pub fn groupby(self, by: Vec<Expr>) -> LazyGroupBy

Group by and aggregate.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

fn example(df: DataFrame) -> LazyFrame {
      df.lazy()
       .groupby(vec![col("date")])
       .agg(vec![
           col("rain").min(),
           col("rain").sum(),
           col("rain").quantile(0.5).alias("median_rain"),
       ])
       .sort("date", false)
}

pub fn left_join(
    self,
    other: LazyFrame,
    left_on: Expr,
    right_on: Expr
) -> LazyFrame

Join query with other lazy query.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;
fn join_dataframes(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
        ldf
        .left_join(other, col("foo"), col("bar"))
}

pub fn outer_join(
    self,
    other: LazyFrame,
    left_on: Expr,
    right_on: Expr
) -> LazyFrame

Join query with other lazy query.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;
fn join_dataframes(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
        ldf
        .outer_join(other, col("foo"), col("bar"))
}

pub fn inner_join(
    self,
    other: LazyFrame,
    left_on: Expr,
    right_on: Expr
) -> LazyFrame

Join query with other lazy query.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;
fn join_dataframes(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
        ldf
        .inner_join(other, col("foo"), col("bar").cast(DataType::Utf8))
}

pub fn join(
    self,
    other: LazyFrame,
    left_on: Vec<Expr>,
    right_on: Vec<Expr>,
    how: JoinType
) -> LazyFrame

Generic join function that can join on multiple columns.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;

fn example(ldf: LazyFrame, other: LazyFrame) -> LazyFrame {
        ldf
        .join(other, vec![col("foo"), col("bar")], vec![col("foo"), col("bar")], JoinType::Inner)
}

pub fn join_builder(self) -> JoinBuilder

pub fn with_column(self, expr: Expr) -> LazyFrame

Add a column to a DataFrame

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;
fn add_column(df: DataFrame) -> LazyFrame {
    df.lazy()
        .with_column(
            when(col("sepal.length").lt(lit(5.0)))
            .then(lit(10))
            .otherwise(lit(1))
            .alias("new_column_name"),
            )
}

pub fn with_columns(self, exprs: Vec<Expr>) -> LazyFrame

Add multiple columns to a DataFrame.

Example

use polars_core::prelude::*;
use polars_lazy::prelude::*;
fn add_columns(df: DataFrame) -> LazyFrame {
    df.lazy()
        .with_columns(
            vec![lit(10).alias("foo"), lit(100).alias("bar")]
         )
}

pub fn max(self) -> LazyFrame

Aggregate all the columns as their maximum values.

pub fn min(self) -> LazyFrame

Aggregate all the columns as their minimum values.

pub fn sum(self) -> LazyFrame

Aggregate all the columns as their sum values.

pub fn mean(self) -> LazyFrame

Aggregate all the columns as their mean values.

pub fn median(self) -> LazyFrame

Aggregate all the columns as their median values.

pub fn quantile(self, quantile: f64) -> LazyFrame

Aggregate all the columns as their quantile values.

pub fn std(self) -> LazyFrame

Aggregate all the columns as their standard deviation values.

pub fn var(self) -> LazyFrame

Aggregate all the columns as their variance values.

pub fn explode(self, columns: &[Expr]) -> LazyFrame

Apply explode operation. See eager explode.

pub fn drop_duplicates(
    self,
    maintain_order: bool,
    subset: Option<Vec<String>>
) -> LazyFrame

Drop duplicate rows. See eager.

pub fn drop_nulls(self, subset: Option<Vec<Expr>>) -> LazyFrame

Drop null rows.

Equal to LazyFrame::filter(col("*").is_not_null())

pub fn slice(self, offset: i64, len: usize) -> LazyFrame

Slice the DataFrame.

pub fn first(self) -> LazyFrame

Get the first row.

pub fn last(self) -> LazyFrame

Get the last row

pub fn tail(self, n: usize) -> LazyFrame

Get the n last rows

pub fn melt(self, id_vars: Vec<String>, value_vars: Vec<String>) -> LazyFrame

Melt the DataFrame from wide to long format

pub fn limit(self, n: usize) -> LazyFrame

Limit the DataFrame to the first n rows. Note if you don’t want the rows to be scanned, use fetch.

pub fn map<F>(
    self,
    function: F,
    optimizations: Option<AllowedOptimizations>,
    schema: Option<Schema>
) -> LazyFrame where
    F: DataFrameUdf + 'static, 

Apply a function/closure once the logical plan get executed.

Warning

This can blow up in your face if the schema is changed due to the operation. The optimizer relies on a correct schema.

You can toggle certain optimizations off.

Trait Implementations

impl Clone for LazyFrame

fn clone(&self) -> LazyFrame

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Default for LazyFrame

fn default() -> Self

Returns the “default value” for a type.

impl From<LogicalPlan> for LazyFrame

fn from(plan: LogicalPlan) -> Self

Performs the conversion.