Struct datafusion_python::context::PySessionContext

pub struct PySessionContext {
    pub ctx: SessionContext,
}

PySessionContext is able to plan and execute DataFusion plans. It has a powerful optimizer, a physical planner for local execution, and a multi-threaded execution engine to perform the execution.

Fields

ctx: SessionContext

Implementations

impl PySessionContext

pub fn new( config: Option<PySessionConfig>, runtime: Option<PyRuntimeConfig>, ) -> PyResult<Self>

pub fn register_object_store( &mut self, scheme: &str, store: StorageContexts, host: Option<&str>, ) -> PyResult<()>

Register an object store with the given name

pub fn register_listing_table( &mut self, name: &str, path: &str, table_partition_cols: Vec<(String, String)>, file_extension: &str, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PyExpr>>>, py: Python<'_>, ) -> PyResult<()>

pub fn sql(&mut self, query: &str, py: Python<'_>) -> PyResult<PyDataFrame>

Returns a PyDataFrame whose plan corresponds to the SQL statement.

pub fn sql_with_options( &mut self, query: &str, options: Option<PySQLOptions>, py: Python<'_>, ) -> PyResult<PyDataFrame>

pub fn create_dataframe( &mut self, partitions: PyArrowType<Vec<Vec<RecordBatch>>>, name: Option<&str>, schema: Option<PyArrowType<Schema>>, py: Python<'_>, ) -> PyResult<PyDataFrame>

pub fn create_dataframe_from_logical_plan( &mut self, plan: PyLogicalPlan, ) -> PyDataFrame

Create a DataFrame from an existing logical plan

pub fn from_pylist( &mut self, data: Bound<'_, PyList>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from Python list

pub fn from_pydict( &mut self, data: Bound<'_, PyDict>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from Python dictionary

pub fn from_arrow_table( &mut self, data: Bound<'_, PyAny>, name: Option<&str>, py: Python<'_>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from Arrow Table

pub fn from_pandas( &mut self, data: Bound<'_, PyAny>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from pandas

pub fn from_polars( &mut self, data: Bound<'_, PyAny>, name: Option<&str>, ) -> PyResult<PyDataFrame>

Construct datafusion dataframe from polars

pub fn register_table(&mut self, name: &str, table: &PyTable) -> PyResult<()>

pub fn deregister_table(&mut self, name: &str) -> PyResult<()>

pub fn register_record_batches( &mut self, name: &str, partitions: PyArrowType<Vec<Vec<RecordBatch>>>, ) -> PyResult<()>

pub fn register_parquet( &mut self, name: &str, path: &str, table_partition_cols: Vec<(String, String)>, parquet_pruning: bool, file_extension: &str, skip_metadata: bool, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PyExpr>>>, py: Python<'_>, ) -> PyResult<()>

pub fn register_csv( &mut self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, has_header: bool, delimiter: &str, schema_infer_max_records: usize, file_extension: &str, file_compression_type: Option<String>, py: Python<'_>, ) -> PyResult<()>

pub fn register_json( &mut self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, String)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyResult<()>

pub fn register_avro( &mut self, name: &str, path: PathBuf, schema: Option<PyArrowType<Schema>>, file_extension: &str, table_partition_cols: Vec<(String, String)>, py: Python<'_>, ) -> PyResult<()>

pub fn register_dataset( &self, name: &str, dataset: &Bound<'_, PyAny>, py: Python<'_>, ) -> PyResult<()>

pub fn register_udf(&mut self, udf: PyScalarUDF) -> PyResult<()>

pub fn register_udaf(&mut self, udaf: PyAggregateUDF) -> PyResult<()>

pub fn catalog(&self, name: &str) -> PyResult<PyCatalog>

pub fn tables(&self) -> HashSet<String>

pub fn table(&self, name: &str, py: Python<'_>) -> PyResult<PyDataFrame>

pub fn table_exist(&self, name: &str) -> PyResult<bool>

pub fn empty_table(&self) -> PyResult<PyDataFrame>

pub fn session_id(&self) -> String

pub fn read_json( &mut self, path: PathBuf, schema: Option<PyArrowType<Schema>>, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, String)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyResult<PyDataFrame>

pub fn read_csv( &self, path: PathBuf, schema: Option<PyArrowType<Schema>>, has_header: bool, delimiter: &str, schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, String)>, file_compression_type: Option<String>, py: Python<'_>, ) -> PyResult<PyDataFrame>

pub fn read_parquet( &self, path: &str, table_partition_cols: Vec<(String, String)>, parquet_pruning: bool, file_extension: &str, skip_metadata: bool, schema: Option<PyArrowType<Schema>>, file_sort_order: Option<Vec<Vec<PyExpr>>>, py: Python<'_>, ) -> PyResult<PyDataFrame>

pub fn read_avro( &self, path: &str, schema: Option<PyArrowType<Schema>>, table_partition_cols: Vec<(String, String)>, file_extension: &str, py: Python<'_>, ) -> PyResult<PyDataFrame>

pub fn read_table(&self, table: &PyTable) -> PyResult<PyDataFrame>

pub fn execute( &self, plan: PyExecutionPlan, part: usize, py: Python<'_>, ) -> PyResult<PyRecordBatchStream>

Execute a partition of an execution plan and return a stream of record batches

Trait Implementations

impl Clone for PySessionContext

fn clone(&self) -> PySessionContext

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl From<PySessionContext> for SessionContext

fn from(ctx: PySessionContext) -> SessionContext

Converts to this type from the input type.

impl From<SessionContext> for PySessionContext

fn from(ctx: SessionContext) -> PySessionContext

Converts to this type from the input type.

impl HasPyGilRef for PySessionContext

type AsRefTarget = PyCell<PySessionContext>

Utility type to make Py::as_ref work.

impl IntoPy<Py<PyAny>> for PySessionContext

fn into_py(self, py: Python<'_>) -> PyObject

Performs the conversion.

impl PyClass for PySessionContext

type Frozen = False

Whether the pyclass is frozen.

impl PyClassImpl for PySessionContext

const IS_BASETYPE: bool = true

#[pyclass(subclass)]

const IS_SUBCLASS: bool = false

#[pyclass(extends=…)]

const IS_MAPPING: bool = false

#[pyclass(mapping)]

const IS_SEQUENCE: bool = false

#[pyclass(sequence)]

type BaseType = PyAny

Base class

type ThreadChecker = SendablePyClass<PySessionContext>

This handles following two situations:

type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::MutableChild

Immutable or mutable

type Dict = PyClassDummySlot

Specify this class has #[pyclass(dict)] or not.

type WeakRef = PyClassDummySlot

Specify this class has #[pyclass(weakref)] or not.

type BaseNativeType = PyAny

The closest native ancestor. This is PyAny by default, and when you declare #[pyclass(extends=PyDict)], it’s PyDict.

fn items_iter() -> PyClassItemsIter

fn doc(py: Python<'_>) -> PyResult<&'static CStr>

Rendered class doc

fn lazy_type_object() -> &'static LazyTypeObject<Self>

fn dict_offset() -> Option<isize>

fn weaklist_offset() -> Option<isize>

impl PyClassNewTextSignature<PySessionContext> for PyClassImplCollector<PySessionContext>

fn new_text_signature(self) -> Option<&'static str>

impl<'a, 'py> PyFunctionArgument<'a, 'py> for &'a PySessionContext

type Holder = Option<PyRef<'py, PySessionContext>>

fn extract( obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder, ) -> PyResult<Self>

impl<'a, 'py> PyFunctionArgument<'a, 'py> for &'a mut PySessionContext

type Holder = Option<PyRefMut<'py, PySessionContext>>

fn extract( obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder, ) -> PyResult<Self>

impl PyMethods<PySessionContext> for PyClassImplCollector<PySessionContext>

fn py_methods(self) -> &'static PyClassItems

impl PyTypeInfo for PySessionContext

const NAME: &'static str = "SessionContext"

Class name.

const MODULE: Option<&'static str> = _

Module name, if any.

fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject

Returns the PyTypeObject instance for this type.

fn type_object(py: Python<'_>) -> &PyType

👎Deprecated since 0.21.0: PyTypeInfo::type_object will be replaced by PyTypeInfo::type_object_bound in a future PyO3 version

Returns the safe abstraction over the type object.

fn type_object_bound(py: Python<'_>) -> Bound<'_, PyType>

Returns the safe abstraction over the type object.

fn is_type_of(object: &PyAny) -> bool

👎Deprecated since 0.21.0: PyTypeInfo::is_type_of will be replaced by PyTypeInfo::is_type_of_bound in a future PyO3 version

Checks if object is an instance of this type or a subclass of this type.

fn is_type_of_bound(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type or a subclass of this type.

fn is_exact_type_of(object: &PyAny) -> bool

👎Deprecated since 0.21.0: PyTypeInfo::is_exact_type_of will be replaced by PyTypeInfo::is_exact_type_of_bound in a future PyO3 version

Checks if object is an instance of this type.

fn is_exact_type_of_bound(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type.

impl DerefToPyAny for PySessionContext