datafusion_python/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// Re-export Apache Arrow DataFusion dependencies
19pub use datafusion::{
20    self, common as datafusion_common, logical_expr as datafusion_expr, optimizer,
21    sql as datafusion_sql,
22};
23#[cfg(feature = "substrait")]
24pub use datafusion_substrait;
25#[cfg(feature = "mimalloc")]
26use mimalloc::MiMalloc;
27use pyo3::prelude::*;
28
29pub mod analyzer;
30pub mod catalog;
31pub mod codec;
32pub mod common;
33
34pub mod context;
35pub mod dataframe;
36mod dataset;
37mod dataset_exec;
38pub mod errors;
39pub mod expr;
40mod functions;
41pub mod metrics;
42mod options;
43pub mod physical_plan;
44mod pyarrow_filter_expression;
45pub mod pyarrow_util;
46mod record_batch;
47mod spark_functions;
48pub mod sql;
49pub mod store;
50pub mod table;
51pub mod unparser;
52
53mod array;
54#[cfg(feature = "substrait")]
55pub mod substrait;
56mod udaf;
57mod udf;
58pub mod udtf;
59mod udwf;
60
61// Re-export helpers previously consumed by downstream Rust crates.
62// Modules stay private to keep the public Rust API surface small.
63pub use udaf::to_rust_accumulator;
64pub use udwf::{MultiColumnWindowUDF, PythonFunctionWindowUDF, to_rust_partition_evaluator};
65
66#[cfg(feature = "mimalloc")]
67#[global_allocator]
68static GLOBAL: MiMalloc = MiMalloc;
69
70/// Low-level DataFusion internal package.
71///
72/// The higher-level public API is defined in pure python files under the
73/// datafusion directory.
74#[pymodule]
75fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
76    // Initialize logging
77    pyo3_log::init();
78
79    // Register the python classes
80    m.add_class::<context::PyRuntimeEnvBuilder>()?;
81    m.add_class::<context::PySessionConfig>()?;
82    m.add_class::<context::PySessionContext>()?;
83    m.add_class::<context::PySQLOptions>()?;
84    m.add_class::<dataframe::PyDataFrame>()?;
85    m.add_class::<dataframe::PyInsertOp>()?;
86    m.add_class::<dataframe::PyDataFrameWriteOptions>()?;
87    m.add_class::<dataframe::PyParquetColumnOptions>()?;
88    m.add_class::<dataframe::PyParquetWriterOptions>()?;
89    m.add_class::<udf::PyScalarUDF>()?;
90    m.add_class::<udaf::PyAggregateUDF>()?;
91    m.add_class::<udwf::PyWindowUDF>()?;
92    m.add_class::<udtf::PyTableFunction>()?;
93    m.add_class::<sql::logical::PyLogicalPlan>()?;
94    m.add_class::<metrics::PyMetricsSet>()?;
95    m.add_class::<metrics::PyMetric>()?;
96    m.add_class::<physical_plan::PyExecutionPlan>()?;
97    m.add_class::<record_batch::PyRecordBatch>()?;
98    m.add_class::<record_batch::PyRecordBatchStream>()?;
99
100    let catalog = PyModule::new(py, "catalog")?;
101    catalog::init_module(&catalog)?;
102    m.add_submodule(&catalog)?;
103
104    // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/
105    let common = PyModule::new(py, "common")?;
106    common::init_module(&common)?;
107    m.add_submodule(&common)?;
108
109    // Register `expr` as a submodule. Matching `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/
110    let expr = PyModule::new(py, "expr")?;
111    expr::init_module(&expr)?;
112    m.add_submodule(&expr)?;
113
114    let unparser = PyModule::new(py, "unparser")?;
115    unparser::init_module(&unparser)?;
116    m.add_submodule(&unparser)?;
117
118    // Register the functions as a submodule
119    let funcs = PyModule::new(py, "functions")?;
120    functions::init_module(&funcs)?;
121    // Spark-compatible functions live under `functions.spark`.
122    let spark_funcs = PyModule::new(py, "spark")?;
123    spark_functions::init_module(&spark_funcs)?;
124    funcs.add_submodule(&spark_funcs)?;
125    m.add_submodule(&funcs)?;
126
127    let store = PyModule::new(py, "object_store")?;
128    store::init_module(&store)?;
129    m.add_submodule(&store)?;
130
131    let options = PyModule::new(py, "options")?;
132    options::init_module(&options)?;
133    m.add_submodule(&options)?;
134
135    // Register substrait as a submodule
136    #[cfg(feature = "substrait")]
137    setup_substrait_module(py, &m)?;
138
139    Ok(())
140}
141
142#[cfg(feature = "substrait")]
143fn setup_substrait_module(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
144    let substrait = PyModule::new(py, "substrait")?;
145    substrait::init_module(&substrait)?;
146    m.add_submodule(&substrait)?;
147    Ok(())
148}
datafusion_python/lib.rs

datafusion_python/
lib.rs
