datafusion_python/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// Re-export Apache Arrow DataFusion dependencies
19pub use datafusion;
20pub use datafusion::{
21    common as datafusion_common, logical_expr as datafusion_expr, optimizer, sql as datafusion_sql,
22};
23#[cfg(feature = "substrait")]
24pub use datafusion_substrait;
25#[cfg(feature = "mimalloc")]
26use mimalloc::MiMalloc;
27use pyo3::prelude::*;
28
29#[allow(clippy::borrow_deref_ref)]
30pub mod catalog;
31pub mod common;
32
33#[allow(clippy::borrow_deref_ref)]
34mod config;
35#[allow(clippy::borrow_deref_ref)]
36pub mod context;
37#[allow(clippy::borrow_deref_ref)]
38pub mod dataframe;
39mod dataset;
40mod dataset_exec;
41pub mod errors;
42#[allow(clippy::borrow_deref_ref)]
43pub mod expr;
44#[allow(clippy::borrow_deref_ref)]
45mod functions;
46mod options;
47pub mod physical_plan;
48mod pyarrow_filter_expression;
49pub mod pyarrow_util;
50mod record_batch;
51pub mod sql;
52pub mod store;
53pub mod table;
54pub mod unparser;
55
56mod array;
57#[cfg(feature = "substrait")]
58pub mod substrait;
59#[allow(clippy::borrow_deref_ref)]
60mod udaf;
61#[allow(clippy::borrow_deref_ref)]
62mod udf;
63pub mod udtf;
64mod udwf;
65pub mod utils;
66
67#[cfg(feature = "mimalloc")]
68#[global_allocator]
69static GLOBAL: MiMalloc = MiMalloc;
70
71// Used to define Tokio Runtime as a Python module attribute
72pub(crate) struct TokioRuntime(tokio::runtime::Runtime);
73
74/// Low-level DataFusion internal package.
75///
76/// The higher-level public API is defined in pure python files under the
77/// datafusion directory.
78#[pymodule]
79fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
80    // Initialize logging
81    pyo3_log::init();
82
83    // Register the python classes
84    m.add_class::<context::PyRuntimeEnvBuilder>()?;
85    m.add_class::<context::PySessionConfig>()?;
86    m.add_class::<context::PySessionContext>()?;
87    m.add_class::<context::PySQLOptions>()?;
88    m.add_class::<dataframe::PyDataFrame>()?;
89    m.add_class::<dataframe::PyInsertOp>()?;
90    m.add_class::<dataframe::PyDataFrameWriteOptions>()?;
91    m.add_class::<dataframe::PyParquetColumnOptions>()?;
92    m.add_class::<dataframe::PyParquetWriterOptions>()?;
93    m.add_class::<udf::PyScalarUDF>()?;
94    m.add_class::<udaf::PyAggregateUDF>()?;
95    m.add_class::<udwf::PyWindowUDF>()?;
96    m.add_class::<udtf::PyTableFunction>()?;
97    m.add_class::<config::PyConfig>()?;
98    m.add_class::<sql::logical::PyLogicalPlan>()?;
99    m.add_class::<physical_plan::PyExecutionPlan>()?;
100    m.add_class::<record_batch::PyRecordBatch>()?;
101    m.add_class::<record_batch::PyRecordBatchStream>()?;
102
103    let catalog = PyModule::new(py, "catalog")?;
104    catalog::init_module(&catalog)?;
105    m.add_submodule(&catalog)?;
106
107    // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/
108    let common = PyModule::new(py, "common")?;
109    common::init_module(&common)?;
110    m.add_submodule(&common)?;
111
112    // Register `expr` as a submodule. Matching `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/
113    let expr = PyModule::new(py, "expr")?;
114    expr::init_module(&expr)?;
115    m.add_submodule(&expr)?;
116
117    let unparser = PyModule::new(py, "unparser")?;
118    unparser::init_module(&unparser)?;
119    m.add_submodule(&unparser)?;
120
121    // Register the functions as a submodule
122    let funcs = PyModule::new(py, "functions")?;
123    functions::init_module(&funcs)?;
124    m.add_submodule(&funcs)?;
125
126    let store = PyModule::new(py, "object_store")?;
127    store::init_module(&store)?;
128    m.add_submodule(&store)?;
129
130    let options = PyModule::new(py, "options")?;
131    options::init_module(&options)?;
132    m.add_submodule(&options)?;
133
134    // Register substrait as a submodule
135    #[cfg(feature = "substrait")]
136    setup_substrait_module(py, &m)?;
137
138    Ok(())
139}
140
141#[cfg(feature = "substrait")]
142fn setup_substrait_module(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
143    let substrait = PyModule::new(py, "substrait")?;
144    substrait::init_module(&substrait)?;
145    m.add_submodule(&substrait)?;
146    Ok(())
147}
datafusion_python/lib.rs

datafusion_python/
lib.rs
