Struct PySessionContext 

pub struct PySessionContext {
    pub ctx: SessionContext,
}

PySessionContext is able to plan and execute DataFusion plans. It has a powerful optimizer, a physical planner for local execution, and a multi-threaded execution engine to perform the execution.

Fields

ctx: SessionContext

Implementations

impl PySessionContext

pub fn new( config: Option<PySessionConfig>, runtime: Option<PyRuntimeEnvBuilder>, ) -> PyDataFusionResult<Self>

pub fn enable_url_table(&self) -> PyResult<Self>

pub fn register_object_store( &self, scheme: &str, store: StorageContexts, host: Option<&str>, ) -> PyResult<()>

Register an object store with the given name

pub fn register_listing_table( &self, name: &str, path: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_extension: &str, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PySortExpr>>>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_udtf(&self, func: PyTableFunction)

pub fn sql( &self, query: &str, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

Returns a PyDataFrame whose plan corresponds to the SQL statement.

pub fn sql_with_options( &self, query: &str, options: Option<PySQLOptions>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn create_dataframe( &self, partitions: PyArrowType<Vec<Vec<RecordBatch>>>, name: Option<&str>, schema: Option<PyArrowType<Schema>>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn create_dataframe_from_logical_plan( &self, plan: PyLogicalPlan, ) -> PyDataFrame

Create a DataFrame from an existing logical plan

pub fn from_pylist( &self, data: Bound<'_, PyList>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from Python list

pub fn from_pydict( &self, data: Bound<'_, PyDict>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from Python dictionary

pub fn from_arrow( &self, data: Bound<'_, PyAny>, name: Option<&str>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

Construct datafusion dataframe from Arrow Table

pub fn from_pandas( &self, data: Bound<'_, PyAny>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from pandas

pub fn from_polars( &self, data: Bound<'_, PyAny>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from polars

pub fn register_table( &self, name: &str, table: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

pub fn deregister_table(&self, name: &str) -> PyDataFusionResult<()>

pub fn register_catalog_provider( &self, name: &str, provider: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

pub fn register_table_provider( &self, name: &str, provider: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

Construct datafusion dataframe from Arrow Table

pub fn register_record_batches( &self, name: &str, partitions: PyArrowType<Vec<Vec<RecordBatch>>>, ) -> PyDataFusionResult<()>

pub fn register_parquet( &self, name: &str, path: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, parquet_pruning: bool, file_extension: &str, skip_metadata: bool, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PySortExpr>>>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_csv( &self, name: &str, path: &Bound<'_, PyAny>, schema: Option<PyArrowType<Schema>>, has_header: bool, delimiter: &str, schema_infer_max_records: usize, file_extension: &str, file_compression_type: Option<String>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_json( &self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_avro( &self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_dataset( &self, name: &str, dataset: &Bound<'_, PyAny>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_udf(&self, udf: PyScalarUDF) -> PyResult<()>

pub fn register_udaf(&self, udaf: PyAggregateUDF) -> PyResult<()>

pub fn register_udwf(&self, udwf: PyWindowUDF) -> PyResult<()>

pub fn catalog(&self, name: &str) -> PyResult<PyObject>

pub fn catalog_names(&self) -> HashSet<String>

pub fn tables(&self) -> HashSet<String>

pub fn table(&self, name: &str, py: Python<'_>) -> PyResult<PyDataFrame>

pub fn table_exist(&self, name: &str) -> PyDataFusionResult<bool>

pub fn empty_table(&self) -> PyDataFusionResult<PyDataFrame>

pub fn session_id(&self) -> String

pub fn read_json( &self, path: PathBuf, schema: Option<PyArrowType<Schema>>, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_csv( &self, path: &Bound<'_, PyAny>, schema: Option<PyArrowType<Schema>>, has_header: bool, delimiter: &str, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_parquet( &self, path: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, parquet_pruning: bool, file_extension: &str, skip_metadata: bool, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PySortExpr>>>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_avro( &self, path: &str, schema: Option<PyArrowType<Schema>>, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_extension: &str, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_table( &self, table: Bound<'_, PyAny>, ) -> PyDataFusionResult<PyDataFrame>

pub fn execute( &self, plan: PyExecutionPlan, part: usize, py: Python<'_>, ) -> PyDataFusionResult<PyRecordBatchStream>

Execute a partition of an execution plan and return a stream of record batches

Trait Implementations

impl Clone for PySessionContext

fn clone(&self) -> PySessionContext

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl From<PySessionContext> for SessionContext

fn from(ctx: PySessionContext) -> SessionContext

Converts to this type from the input type.

impl From<SessionContext> for PySessionContext

fn from(ctx: SessionContext) -> PySessionContext

Converts to this type from the input type.

impl<'py> IntoPyObject<'py> for PySessionContext

type Target = PySessionContext

The Python output type

type Output = Bound<'py, <PySessionContext as IntoPyObject<'py>>::Target>

The smart pointer type to use.

type Error = PyErr

The type returned in the event of a conversion error.

fn into_pyobject( self, py: Python<'py>, ) -> Result<<Self as IntoPyObject<'_>>::Output, <Self as IntoPyObject<'_>>::Error>

Performs the conversion.

impl PyClass for PySessionContext

type Frozen = True

Whether the pyclass is frozen.

impl PyClassBaseType for PySessionContext

type LayoutAsBase = PyClassObject<PySessionContext>

type BaseNativeType = <PySessionContext as PyClassImpl>::BaseNativeType

type Initializer = PyClassInitializer<PySessionContext>

type PyClassMutability = <PySessionContext as PyClassImpl>::PyClassMutability

impl PyClassImpl for PySessionContext

const IS_BASETYPE: bool = true

#[pyclass(subclass)]

const IS_SUBCLASS: bool = false

#[pyclass(extends=…)]

const IS_MAPPING: bool = false

#[pyclass(mapping)]

const IS_SEQUENCE: bool = false

#[pyclass(sequence)]

const IS_IMMUTABLE_TYPE: bool = false

#[pyclass(immutable_type)]

type BaseType = PyAny

Base class

type ThreadChecker = SendablePyClass<PySessionContext>

This handles following two situations:

type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::ImmutableChild

Immutable or mutable

type Dict = PyClassDummySlot

Specify this class has #[pyclass(dict)] or not.

type WeakRef = PyClassDummySlot

Specify this class has #[pyclass(weakref)] or not.

type BaseNativeType = PyAny

The closest native ancestor. This is PyAny by default, and when you declare #[pyclass(extends=PyDict)], it’s PyDict.

fn items_iter() -> PyClassItemsIter

fn doc(py: Python<'_>) -> PyResult<&'static CStr>

Rendered class doc

fn lazy_type_object() -> &'static LazyTypeObject<Self>

fn dict_offset() -> Option<isize>

fn weaklist_offset() -> Option<isize>

impl PyClassNewTextSignature<PySessionContext> for PyClassImplCollector<PySessionContext>

fn new_text_signature(self) -> Option<&'static str>

impl<'a, 'py> PyFunctionArgument<'a, 'py, false> for &'a PySessionContext

type Holder = Option<PyRef<'py, PySessionContext>>

fn extract( obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder, ) -> PyResult<Self>

impl PyMethods<PySessionContext> for PyClassImplCollector<PySessionContext>

fn py_methods(self) -> &'static PyClassItems

impl PyTypeInfo for PySessionContext

const NAME: &'static str = "SessionContext"

Class name.

const MODULE: Option<&'static str>

Module name, if any.

fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject

Returns the PyTypeObject instance for this type.

fn type_object(py: Python<'_>) -> Bound<'_, PyType>

Returns the safe abstraction over the type object.

fn is_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type or a subclass of this type.

fn is_exact_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type.

impl DerefToPyAny for PySessionContext