datafusion_substrait/lib.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#![doc(
19 html_logo_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg",
20 html_favicon_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg"
21)]
22#![cfg_attr(docsrs, feature(doc_cfg))]
23// Make sure fast / cheap clones on Arc are explicit:
24// https://github.com/apache/datafusion/issues/11143
25#![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]
26#![deny(clippy::allow_attributes)]
27#![cfg_attr(test, allow(clippy::needless_pass_by_value))]
28
29//! Serialize / Deserialize DataFusion Plans to [Substrait.io]
30//!
31//! This crate provides support for serializing and deserializing both DataFusion
32//! [`LogicalPlan`] and [`ExecutionPlan`] to and from the generated types in
33//! [substrait::proto] from the [substrait] crate.
34//!
35//! [Substrait.io] provides a cross-language serialization format for relational
36//! algebra (e.g. query plans and expressions), based on protocol buffers.
37//!
38//! [Substrait.io]: https://substrait.io/
39//!
40//! [`LogicalPlan`]: datafusion::logical_expr::LogicalPlan
41//! [`ExecutionPlan`]: datafusion::physical_plan::ExecutionPlan
42//!
43//! Potential uses of this crate:
44//! * Use DataFusion to run Substrait plans created by other systems (e.g. Apache Calcite)
45//! * Use DataFusion to create plans to run on other systems
46//! * Pass query plans over FFI boundaries, such as from Python to Rust
47//! * Pass query plans across node boundaries
48//!
49//! # See Also
50//!
51//! Substrait does not (yet) support the full range of plans and expressions
52//! that DataFusion offers. See the [datafusion-proto] crate for a DataFusion
53//! specific format that does support of the full range.
54//!
55//! [datafusion-proto]: https://docs.rs/datafusion-proto/latest/datafusion_proto
56//!
57//! Note that generated types such as [`substrait::proto::Plan`] and
58//! [`substrait::proto::Rel`] can be serialized / deserialized to bytes, JSON and
59//! other formats using [prost] and the rest of the Rust protobuf ecosystem.
60//!
61//! # Example: Serializing [`LogicalPlan`]s
62//! ```
63//! # use datafusion::prelude::*;
64//! # use datafusion::error::Result;
65//! # #[tokio::main(flavor = "current_thread")]
66//! # async fn main() -> Result<()>{
67//! # use std::sync::Arc;
68//! # use datafusion::arrow::array::{Int32Array, RecordBatch};
69//! # use datafusion_substrait::logical_plan;
70//! // Create a plan that scans table 't'
71//! let ctx = SessionContext::new();
72//! let batch = RecordBatch::try_from_iter(vec![(
73//! "x",
74//! Arc::new(Int32Array::from(vec![42])) as _,
75//! )])?;
76//! ctx.register_batch("t", batch)?;
77//! let df = ctx.sql("SELECT x from t").await?;
78//! let plan = df.into_optimized_plan()?;
79//!
80//! // Convert the plan into a substrait (protobuf) Plan
81//! let substrait_plan = logical_plan::producer::to_substrait_plan(&plan, &ctx.state())?;
82//!
83//! // Receive a substrait protobuf from somewhere, and turn it into a LogicalPlan
84//! let logical_round_trip =
85//! logical_plan::consumer::from_substrait_plan(&ctx.state(), &substrait_plan)
86//! .await?;
87//! let logical_round_trip = ctx.state().optimize(&logical_round_trip)?;
88//! assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip));
89//! # Ok(())
90//! # }
91//! ```
92pub mod extensions;
93pub mod logical_plan;
94#[cfg(feature = "physical")]
95pub mod physical_plan;
96pub mod serializer;
97pub mod variation_const;
98
99// Re-export substrait crate
100pub use substrait;