datafusion_proto/lib.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#![doc(
19 html_logo_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg",
20 html_favicon_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg"
21)]
22#![cfg_attr(docsrs, feature(doc_cfg))]
23// Make sure fast / cheap clones on Arc are explicit:
24// https://github.com/apache/datafusion/issues/11143
25#![deny(clippy::clone_on_ref_ptr)]
26#![cfg_attr(test, allow(clippy::needless_pass_by_value))]
27
28//! Serialize / Deserialize DataFusion Plans to bytes
29//!
30//! This crate provides support for serializing and deserializing the
31//! following structures to and from bytes:
32//!
33//! 1. [`LogicalPlan`]'s (including [`Expr`]),
34//! 2. [`ExecutionPlan`]s (including [`PhysicalExpr`])
35//!
36//! [`LogicalPlan`]: datafusion_expr::LogicalPlan
37//! [`Expr`]: datafusion_expr::Expr
38//! [`ExecutionPlan`]: datafusion_physical_plan::ExecutionPlan
39//! [`PhysicalExpr`]: datafusion_physical_expr::PhysicalExpr
40//!
41//! Internally, this crate is implemented by converting the plans to [protocol
42//! buffers] using [prost].
43//!
44//! [protocol buffers]: https://developers.google.com/protocol-buffers
45//! [prost]: https://docs.rs/prost/latest/prost/
46//!
47//! # Version Compatibility
48//!
49//! The serialized form are not guaranteed to be compatible across
50//! DataFusion versions. A plan serialized with one version of DataFusion
51//! may not be able to deserialized with a different version.
52//!
53//! # See Also
54//!
55//! The binary format created by this crate supports the full range of DataFusion
56//! plans, but is DataFusion specific. See [datafusion-substrait] for a crate
57//! which can encode many DataFusion plans using the [substrait.io] standard.
58//!
59//! [datafusion-substrait]: https://docs.rs/datafusion-substrait/latest/datafusion_substrait
60//! [substrait.io]: https://substrait.io
61//!
62//! # Example: Serializing [`Expr`]s
63//! ```
64//! # use datafusion_common::Result;
65//! # use datafusion_expr::{col, lit, Expr};
66//! # use datafusion_proto::bytes::Serializeable;
67//! # fn main() -> Result<()>{
68//! // Create a new `Expr` a < 32
69//! let expr = col("a").lt(lit(5i32));
70//!
71//! // Convert it to bytes (for sending over the network, etc.)
72//! let bytes = expr.to_bytes()?;
73//!
74//! // Decode bytes from somewhere (over network, etc.) back to Expr
75//! let decoded_expr = Expr::from_bytes(&bytes)?;
76//! assert_eq!(expr, decoded_expr);
77//! # Ok(())
78//! # }
79//! ```
80//!
81//! # Example: Serializing [`LogicalPlan`]s
82//! ```
83//! # use datafusion::prelude::*;
84//! # use datafusion_common::Result;
85//! # use datafusion_proto::bytes::{logical_plan_from_bytes, logical_plan_to_bytes};
86//! # #[tokio::main]
87//! # async fn main() -> Result<()>{
88//! // Create a plan that scans table 't'
89//! let ctx = SessionContext::new();
90//! ctx.register_csv("t1", "tests/testdata/test.csv", CsvReadOptions::default()).await?;
91//! let plan = ctx.table("t1").await?.into_optimized_plan()?;
92//!
93//! // Convert the plan into bytes (for sending over the network, etc.)
94//! let bytes = logical_plan_to_bytes(&plan)?;
95//!
96//! // Decode bytes from somewhere (over network, etc.) back to LogicalPlan
97//! let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
98//! assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip));
99//! # Ok(())
100//! # }
101//! ```
102//! # Example: Serializing [`ExecutionPlan`]s
103//!
104//! ```
105//! # use datafusion::prelude::*;
106//! # use datafusion_common::Result;
107//! # use datafusion_proto::bytes::{physical_plan_from_bytes,physical_plan_to_bytes};
108//! # #[tokio::main]
109//! # async fn main() -> Result<()>{
110//! // Create a plan that scans table 't'
111//! let ctx = SessionContext::new();
112//! ctx.register_csv("t1", "tests/testdata/test.csv", CsvReadOptions::default()).await?;
113//! let physical_plan = ctx.table("t1").await?.create_physical_plan().await?;
114//!
115//! // Convert the plan into bytes (for sending over the network, etc.)
116//! let bytes = physical_plan_to_bytes(physical_plan.clone())?;
117//!
118//! // Decode bytes from somewhere (over network, etc.) back to ExecutionPlan
119//! let physical_round_trip = physical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
120//! assert_eq!(format!("{:?}", physical_plan), format!("{:?}", physical_round_trip));
121//! # Ok(())
122//! # }
123//! ```
124pub mod bytes;
125pub mod common;
126pub mod generated;
127pub mod logical_plan;
128pub mod physical_plan;
129
130pub mod protobuf {
131 pub use crate::generated::datafusion::*;
132 pub use datafusion_proto_common::common::proto_error;
133 pub use datafusion_proto_common::protobuf_common::{
134 ArrowFormat, ArrowOptions, ArrowType, AvroFormat, AvroOptions, CsvFormat,
135 DfSchema, EmptyMessage, Field, JoinSide, NdJsonFormat, ParquetFormat,
136 ScalarValue, Schema,
137 };
138 pub use datafusion_proto_common::{FromProtoError, ToProtoError};
139}
140
141#[cfg(doctest)]
142doc_comment::doctest!("../README.md", readme_example_test);