polars-lazy 0.55.2

Lazy query engine for the Polars DataFrame library
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

use std::sync::Arc;

use either::Either;
use polars_core::schema::SchemaRef;
use pyo3::{Py, PyAny};

use self::python_dsl::{PythonOptionsDsl, PythonScanSource};
use crate::prelude::*;

impl LazyFrame {
    pub fn scan_from_python_function(
        schema: Either<Py<PyAny>, SchemaRef>,
        scan_fn: Py<PyAny>,
        pyarrow: bool,
        // Validate that the source gives the proper schema
        validate_schema: bool,
        is_pure: bool,
    ) -> Self {
        DslPlan::PythonScan {
            options: PythonOptionsDsl {
                // Should be a python function that returns a generator
                scan_fn: Some(scan_fn.into()),
                schema_fn: Some(SpecialEq::new(Arc::new(schema.map_left(|obj| obj.into())))),
                python_source: if pyarrow {
                    PythonScanSource::Pyarrow
                } else {
                    PythonScanSource::IOPlugin
                },
                validate_schema,
                is_pure,
            },
        }
        .into()
    }
}