Struct PySessionContext 

pub struct PySessionContext {
    pub ctx: Arc<SessionContext>,
    /* private fields */
}

PySessionContext is able to plan and execute DataFusion plans. It has a powerful optimizer, a physical planner for local execution, and a multi-threaded execution engine to perform the execution.

Fields

ctx: Arc<SessionContext>

Implementations

impl PySessionContext

pub fn new( config: Option<PySessionConfig>, runtime: Option<PyRuntimeEnvBuilder>, ) -> PyDataFusionResult<Self>

pub fn enable_url_table(&self) -> PyResult<Self>

pub fn global_ctx() -> PyResult<Self>

pub fn register_object_store( &self, scheme: &str, store: StorageContexts, host: Option<&str>, ) -> PyResult<()>

Register an object store with the given name

pub fn deregister_object_store( &self, scheme: &str, host: Option<&str>, ) -> PyDataFusionResult<()>

Deregister an object store with the given url

pub fn register_listing_table( &self, name: &str, path: PathBuf, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_extension: &str, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PySortExpr>>>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_udtf(&self, func: PyTableFunction)

pub fn deregister_udtf(&self, name: &str)

pub fn sql_with_options( &self, py: Python<'_>, query: String, options: Option<PySQLOptions>, param_values: HashMap<String, PyScalarValue>, param_strings: HashMap<String, String>, ) -> PyDataFusionResult<PyDataFrame>

pub fn create_dataframe( &self, partitions: PyArrowType<Vec<Vec<RecordBatch>>>, name: Option<&str>, schema: Option<PyArrowType<Schema>>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn create_dataframe_from_logical_plan( &self, plan: PyLogicalPlan, ) -> PyDataFrame

Create a DataFrame from an existing logical plan

pub fn from_pylist( &self, data: Bound<'_, PyList>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from Python list

pub fn from_pydict( &self, data: Bound<'_, PyDict>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from Python dictionary

pub fn from_arrow( &self, data: Bound<'_, PyAny>, name: Option<&str>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

Construct datafusion dataframe from Arrow Table

pub fn from_pandas( &self, data: Bound<'_, PyAny>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from pandas

pub fn from_polars( &self, data: Bound<'_, PyAny>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from polars

pub fn register_table( &self, name: &str, table: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

pub fn deregister_table(&self, name: &str) -> PyDataFusionResult<()>

pub fn register_table_factory( &self, format: &str, factory: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

pub fn register_catalog_provider_list( &self, provider: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

pub fn register_catalog_provider( &self, name: &str, provider: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

pub fn register_table_provider( &self, name: &str, provider: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

Construct datafusion dataframe from Arrow Table

pub fn register_record_batches( &self, name: &str, partitions: PyArrowType<Vec<Vec<RecordBatch>>>, ) -> PyDataFusionResult<()>

pub fn read_batches( &self, batches: PyArrowType<Vec<RecordBatch>>, ) -> PyDataFusionResult<PyDataFrame>

pub fn register_parquet( &self, name: &str, path: PathBuf, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, parquet_pruning: bool, file_extension: &str, skip_metadata: bool, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PySortExpr>>>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_csv( &self, name: &str, path: &Bound<'_, PyAny>, options: Option<&PyCsvReadOptions>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_json( &self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_avro( &self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_arrow( &self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_batch( &self, name: &str, batch: PyArrowType<RecordBatch>, ) -> PyDataFusionResult<()>

pub fn register_dataset( &self, name: &str, dataset: &Bound<'_, PyAny>, py: Python<'_>, ) -> PyDataFusionResult<()>

pub fn register_udf(&self, udf: PyScalarUDF) -> PyResult<()>

pub fn deregister_udf(&self, name: &str)

pub fn register_udaf(&self, udaf: PyAggregateUDF) -> PyResult<()>

pub fn enable_spark_functions(&self) -> PyResult<()>

Register all datafusion-spark UDFs/UDAFs/UDWFs, overriding any built-in DataFusion functions of the same name with their Spark-semantics version.

pub fn deregister_udaf(&self, name: &str)

pub fn register_udwf(&self, udwf: PyWindowUDF) -> PyResult<()>

pub fn deregister_udwf(&self, name: &str)

pub fn udf(&self, name: &str) -> PyResult<PyScalarUDF>

pub fn udaf(&self, name: &str) -> PyResult<PyAggregateUDF>

pub fn udwf(&self, name: &str) -> PyResult<PyWindowUDF>

pub fn udfs(&self) -> Vec<String>

pub fn udafs(&self) -> Vec<String>

pub fn udwfs(&self) -> Vec<String>

pub fn catalog(&self, py: Python<'_>, name: &str) -> PyResult<Py<PyAny>>

pub fn catalog_names(&self) -> HashSet<String>

pub fn table(&self, name: &str, py: Python<'_>) -> PyResult<PyDataFrame>

pub fn table_exist(&self, name: &str) -> PyDataFusionResult<bool>

pub fn empty_table(&self) -> PyDataFusionResult<PyDataFrame>

pub fn session_id(&self) -> String

pub fn copied_config(&self) -> PySessionConfig

Return a copy of the active SessionConfig. Mutating the returned config does not affect this context.

pub fn parse_capacity_limit( config_name: &str, limit: &str, ) -> PyDataFusionResult<usize>

Parse a string like "100M", "1.5G", or "512K" into a byte count. "0" is accepted and returns 0. Use this when constructing a RuntimeEnvBuilder from a human-friendly size string.

pub fn session_start_time(&self) -> String

pub fn enable_ident_normalization(&self) -> bool

pub fn parse_sql_expr( &self, sql: &str, schema: PyDFSchema, ) -> PyDataFusionResult<PyExpr>

pub fn execute_logical_plan( &self, plan: PyLogicalPlan, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn refresh_catalogs(&self, py: Python<'_>) -> PyDataFusionResult<()>

pub fn remove_optimizer_rule(&self, name: &str) -> bool

pub fn add_physical_optimizer_rule( &self, rule: Bound<'_, PyAny>, ) -> PyDataFusionResult<()>

pub fn table_provider(&self, name: &str, py: Python<'_>) -> PyResult<PyTable>

pub fn read_json( &self, path: PathBuf, schema: Option<PyArrowType<Schema>>, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_csv( &self, path: &Bound<'_, PyAny>, options: Option<&PyCsvReadOptions>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_parquet( &self, path: PathBuf, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, parquet_pruning: bool, file_extension: &str, skip_metadata: bool, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PySortExpr>>>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_avro( &self, path: PathBuf, schema: Option<PyArrowType<Schema>>, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, file_extension: &str, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_arrow( &self, path: PathBuf, schema: Option<PyArrowType<Schema>>, file_extension: &str, table_partition_cols: Vec<(String, PyArrowType<DataType>)>, py: Python<'_>, ) -> PyDataFusionResult<PyDataFrame>

pub fn read_table( &self, table: Bound<'_, PyAny>, ) -> PyDataFusionResult<PyDataFrame>

pub fn execute( &self, plan: PyExecutionPlan, part: usize, py: Python<'_>, ) -> PyDataFusionResult<PyRecordBatchStream>

Execute a partition of an execution plan and return a stream of record batches

pub fn __datafusion_task_context_provider__<'py>( &self, py: Python<'py>, ) -> PyResult<Bound<'py, PyCapsule>>

pub fn __datafusion_logical_extension_codec__<'py>( &self, py: Python<'py>, ) -> PyResult<Bound<'py, PyCapsule>>

pub fn with_logical_extension_codec<'py>( &self, codec: Bound<'py, PyAny>, ) -> PyDataFusionResult<Self>

pub fn __datafusion_physical_extension_codec__<'py>( &self, py: Python<'py>, ) -> PyResult<Bound<'py, PyCapsule>>

pub fn with_physical_extension_codec<'py>( &self, codec: Bound<'py, PyAny>, ) -> PyDataFusionResult<Self>

pub fn with_python_udf_inlining(&self, enabled: bool) -> Self

Trait Implementations

impl Clone for PySessionContext

fn clone(&self) -> PySessionContext

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl DerefToPyAny for PySessionContext

impl From<PySessionContext> for SessionContext

fn from(ctx: PySessionContext) -> SessionContext

Converts to this type from the input type.

impl From<SessionContext> for PySessionContext

fn from(ctx: SessionContext) -> PySessionContext

Converts to this type from the input type.

impl<'a, 'py> FromPyObject<'a, 'py> for PySessionContext

where Self: Clone,

type Error = PyClassGuardError<'a, 'py>

The type returned in the event of a conversion error.

fn extract( obj: Borrowed<'a, 'py, PyAny>, ) -> Result<Self, <Self as FromPyObject<'a, 'py>>::Error>

Extracts Self from the bound smart pointer obj.

impl<'py> IntoPyObject<'py> for PySessionContext

type Target = PySessionContext

The Python output type

type Output = Bound<'py, <PySessionContext as IntoPyObject<'py>>::Target>

The smart pointer type to use.

type Error = PyErr

The type returned in the event of a conversion error.

fn into_pyobject( self, py: Python<'py>, ) -> Result<<Self as IntoPyObject<'_>>::Output, <Self as IntoPyObject<'_>>::Error>

Performs the conversion.

impl PyClass for PySessionContext

const NAME: &str = "SessionContext"

Name of the class.

type Frozen = True

Whether the pyclass is frozen.

impl PyClassBaseType for PySessionContext

type LayoutAsBase = <PySessionContext as PyClassImpl>::Layout

type BaseNativeType = <PySessionContext as PyClassImpl>::BaseNativeType

type Initializer = PyClassInitializer<PySessionContext>

type PyClassMutability = <PySessionContext as PyClassImpl>::PyClassMutability

type Layout<T: PyClassImpl> = <<PySessionContext as PyClassBaseType>::BaseNativeType as PyClassBaseType>::Layout<T>

The type of object layout to use for ancestors or descendants of this type.

impl PyClassImpl for PySessionContext

const MODULE: Option<&str>

Module which the class will be associated with.

const IS_BASETYPE: bool = true

#[pyclass(subclass)]

const IS_SUBCLASS: bool = false

#[pyclass(extends=…)]

const IS_MAPPING: bool = false

#[pyclass(mapping)]

const IS_SEQUENCE: bool = false

#[pyclass(sequence)]

const IS_IMMUTABLE_TYPE: bool = false

#[pyclass(immutable_type)]

const RAW_DOC: &'static CStr = /// `PySessionContext` is able to plan and execute DataFusion plans. /// It has a powerful optimizer, a physical planner for local execution, and a /// multi-threaded execution engine to perform the execution.

Docstring for the class provided on the struct or enum.

const DOC: &'static CStr

Fully rendered class doc, including the text_signature if a constructor is defined.

type Layout = <<PySessionContext as PyClassImpl>::BaseNativeType as PyClassBaseType>::Layout<PySessionContext>

Description of how this class is laid out in memory

type BaseType = PyAny

Base class

type ThreadChecker = NoopThreadChecker

This handles following two situations:

type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::ImmutableChild

Immutable or mutable

type Dict = PyClassDummySlot

Specify this class has #[pyclass(dict)] or not.

type WeakRef = PyClassDummySlot

Specify this class has #[pyclass(weakref)] or not.

type BaseNativeType = PyAny

The closest native ancestor. This is PyAny by default, and when you declare #[pyclass(extends=PyDict)], it’s PyDict.

fn items_iter() -> PyClassItemsIter

fn lazy_type_object() -> &'static LazyTypeObject<Self>

fn dict_offset() -> Option<PyObjectOffset>

Used to provide the dictoffset slot (equivalent to tp_dictoffset)

fn weaklist_offset() -> Option<PyObjectOffset>

Used to provide the weaklistoffset slot (equivalent to tp_weaklistoffset

impl PyClassNewTextSignature for PySessionContext

const TEXT_SIGNATURE: &'static str = "(config=None, runtime=None)"

impl PyMethods<PySessionContext> for PyClassImplCollector<PySessionContext>

fn py_methods(self) -> &'static PyClassItems

impl PyTypeInfo for PySessionContext

const NAME: &str = <Self as ::pyo3::PyClass>::NAME

👎Deprecated since 0.28.0:

prefer using ::type_object(py).name() to get the correct runtime value

Class name.

const MODULE: Option<&str> = <Self as ::pyo3::impl_::pyclass::PyClassImpl>::MODULE

👎Deprecated since 0.28.0:

prefer using ::type_object(py).module() to get the correct runtime value

Module name, if any.

fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject

Returns the PyTypeObject instance for this type.

fn type_object(py: Python<'_>) -> Bound<'_, PyType>

Returns the safe abstraction over the type object.

fn is_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type or a subclass of this type.

fn is_exact_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type.