datafusion_python/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#[cfg(feature = "mimalloc")]
19use mimalloc::MiMalloc;
20use pyo3::prelude::*;
21
22// Re-export Apache Arrow DataFusion dependencies
23pub use datafusion;
24pub use datafusion::common as datafusion_common;
25pub use datafusion::logical_expr as datafusion_expr;
26pub use datafusion::optimizer;
27pub use datafusion::sql as datafusion_sql;
28
29#[cfg(feature = "substrait")]
30pub use datafusion_substrait;
31
32#[allow(clippy::borrow_deref_ref)]
33pub mod catalog;
34pub mod common;
35
36#[allow(clippy::borrow_deref_ref)]
37mod config;
38#[allow(clippy::borrow_deref_ref)]
39pub mod context;
40#[allow(clippy::borrow_deref_ref)]
41pub mod dataframe;
42mod dataset;
43mod dataset_exec;
44pub mod errors;
45#[allow(clippy::borrow_deref_ref)]
46pub mod expr;
47#[allow(clippy::borrow_deref_ref)]
48mod functions;
49pub mod physical_plan;
50mod pyarrow_filter_expression;
51pub mod pyarrow_util;
52mod record_batch;
53pub mod sql;
54pub mod store;
55pub mod table;
56pub mod unparser;
57
58#[cfg(feature = "substrait")]
59pub mod substrait;
60#[allow(clippy::borrow_deref_ref)]
61mod udaf;
62#[allow(clippy::borrow_deref_ref)]
63mod udf;
64pub mod udtf;
65mod udwf;
66pub mod utils;
67
68#[cfg(feature = "mimalloc")]
69#[global_allocator]
70static GLOBAL: MiMalloc = MiMalloc;
71
72// Used to define Tokio Runtime as a Python module attribute
73pub(crate) struct TokioRuntime(tokio::runtime::Runtime);
74
75/// Low-level DataFusion internal package.
76///
77/// The higher-level public API is defined in pure python files under the
78/// datafusion directory.
79#[pymodule]
80fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
81    // Initialize logging
82    pyo3_log::init();
83
84    // Register the python classes
85    m.add_class::<context::PyRuntimeEnvBuilder>()?;
86    m.add_class::<context::PySessionConfig>()?;
87    m.add_class::<context::PySessionContext>()?;
88    m.add_class::<context::PySQLOptions>()?;
89    m.add_class::<dataframe::PyDataFrame>()?;
90    m.add_class::<dataframe::PyInsertOp>()?;
91    m.add_class::<dataframe::PyDataFrameWriteOptions>()?;
92    m.add_class::<dataframe::PyParquetColumnOptions>()?;
93    m.add_class::<dataframe::PyParquetWriterOptions>()?;
94    m.add_class::<udf::PyScalarUDF>()?;
95    m.add_class::<udaf::PyAggregateUDF>()?;
96    m.add_class::<udwf::PyWindowUDF>()?;
97    m.add_class::<udtf::PyTableFunction>()?;
98    m.add_class::<config::PyConfig>()?;
99    m.add_class::<sql::logical::PyLogicalPlan>()?;
100    m.add_class::<physical_plan::PyExecutionPlan>()?;
101    m.add_class::<record_batch::PyRecordBatch>()?;
102    m.add_class::<record_batch::PyRecordBatchStream>()?;
103
104    let catalog = PyModule::new(py, "catalog")?;
105    catalog::init_module(&catalog)?;
106    m.add_submodule(&catalog)?;
107
108    // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/
109    let common = PyModule::new(py, "common")?;
110    common::init_module(&common)?;
111    m.add_submodule(&common)?;
112
113    // Register `expr` as a submodule. Matching `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/
114    let expr = PyModule::new(py, "expr")?;
115    expr::init_module(&expr)?;
116    m.add_submodule(&expr)?;
117
118    let unparser = PyModule::new(py, "unparser")?;
119    unparser::init_module(&unparser)?;
120    m.add_submodule(&unparser)?;
121
122    // Register the functions as a submodule
123    let funcs = PyModule::new(py, "functions")?;
124    functions::init_module(&funcs)?;
125    m.add_submodule(&funcs)?;
126
127    let store = PyModule::new(py, "object_store")?;
128    store::init_module(&store)?;
129    m.add_submodule(&store)?;
130
131    // Register substrait as a submodule
132    #[cfg(feature = "substrait")]
133    setup_substrait_module(py, &m)?;
134
135    Ok(())
136}
137
138#[cfg(feature = "substrait")]
139fn setup_substrait_module(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
140    let substrait = PyModule::new(py, "substrait")?;
141    substrait::init_module(&substrait)?;
142    m.add_submodule(&substrait)?;
143    Ok(())
144}
datafusion_python/lib.rs

datafusion_python/
lib.rs
