datafusion_proto/lib.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#![doc(
19 html_logo_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg",
20 html_favicon_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg"
21)]
22#![cfg_attr(docsrs, feature(doc_cfg))]
23// Make sure fast / cheap clones on Arc are explicit:
24// https://github.com/apache/datafusion/issues/11143
25#![deny(clippy::clone_on_ref_ptr)]
26#![deny(clippy::allow_attributes)]
27#![cfg_attr(test, allow(clippy::needless_pass_by_value))]
28
29//! Serialize / Deserialize DataFusion Plans to bytes
30//!
31//! This crate provides support for serializing and deserializing the
32//! following structures to and from bytes:
33//!
34//! 1. [`LogicalPlan`]'s (including [`Expr`]),
35//! 2. [`ExecutionPlan`]s (including [`PhysicalExpr`])
36//!
37//! [`LogicalPlan`]: datafusion_expr::LogicalPlan
38//! [`Expr`]: datafusion_expr::Expr
39//! [`ExecutionPlan`]: datafusion_physical_plan::ExecutionPlan
40//! [`PhysicalExpr`]: datafusion_physical_expr::PhysicalExpr
41//!
42//! Internally, this crate is implemented by converting the plans to [protocol
43//! buffers] using [prost].
44//!
45//! [protocol buffers]: https://developers.google.com/protocol-buffers
46//! [prost]: https://docs.rs/prost/latest/prost/
47//!
48//! # Version Compatibility
49//!
50//! The serialized form are not guaranteed to be compatible across
51//! DataFusion versions. A plan serialized with one version of DataFusion
52//! may not be able to deserialized with a different version.
53//!
54//! # See Also
55//!
56//! The binary format created by this crate supports the full range of DataFusion
57//! plans, but is DataFusion specific. See [datafusion-substrait] for a crate
58//! which can encode many DataFusion plans using the [substrait.io] standard.
59//!
60//! [datafusion-substrait]: https://docs.rs/datafusion-substrait/latest/datafusion_substrait
61//! [substrait.io]: https://substrait.io
62//!
63//! # Example: Serializing [`Expr`]s
64//! ```
65//! # use datafusion_common::Result;
66//! # use datafusion_expr::{col, lit, Expr};
67//! # use datafusion_proto::bytes::Serializeable;
68//! # fn main() -> Result<()>{
69//! // Create a new `Expr` a < 32
70//! let expr = col("a").lt(lit(5i32));
71//!
72//! // Convert it to bytes (for sending over the network, etc.)
73//! let bytes = expr.to_bytes()?;
74//!
75//! // Decode bytes from somewhere (over network, etc.) back to Expr
76//! let decoded_expr = Expr::from_bytes(&bytes)?;
77//! assert_eq!(expr, decoded_expr);
78//! # Ok(())
79//! # }
80//! ```
81//!
82//! # Example: Serializing [`LogicalPlan`]s
83//! ```
84//! # use datafusion::prelude::*;
85//! # use datafusion_common::Result;
86//! # use datafusion_proto::bytes::{logical_plan_from_bytes, logical_plan_to_bytes};
87//! # #[tokio::main]
88//! # async fn main() -> Result<()>{
89//! // Create a plan that scans table 't'
90//! let ctx = SessionContext::new();
91//! ctx.register_csv("t1", "tests/testdata/test.csv", CsvReadOptions::default()).await?;
92//! let plan = ctx.table("t1").await?.into_optimized_plan()?;
93//!
94//! // Convert the plan into bytes (for sending over the network, etc.)
95//! let bytes = logical_plan_to_bytes(&plan)?;
96//!
97//! // Decode bytes from somewhere (over network, etc.) back to LogicalPlan
98//! let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
99//! assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip));
100//! # Ok(())
101//! # }
102//! ```
103//! # Example: Serializing [`ExecutionPlan`]s
104//!
105//! ```
106//! # use datafusion::prelude::*;
107//! # use datafusion_common::Result;
108//! # use datafusion_proto::bytes::{physical_plan_from_bytes,physical_plan_to_bytes};
109//! # #[tokio::main]
110//! # async fn main() -> Result<()>{
111//! // Create a plan that scans table 't'
112//! let ctx = SessionContext::new();
113//! ctx.register_csv("t1", "tests/testdata/test.csv", CsvReadOptions::default()).await?;
114//! let physical_plan = ctx.table("t1").await?.create_physical_plan().await?;
115//!
116//! // Convert the plan into bytes (for sending over the network, etc.)
117//! let bytes = physical_plan_to_bytes(physical_plan.clone())?;
118//!
119//! // Decode bytes from somewhere (over network, etc.) back to ExecutionPlan
120//! let physical_round_trip = physical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
121//! assert_eq!(format!("{:?}", physical_plan), format!("{:?}", physical_round_trip));
122//! # Ok(())
123//! # }
124//! ```
125pub mod bytes;
126pub mod common;
127pub mod generated;
128pub mod logical_plan;
129pub mod physical_plan;
130
131pub mod protobuf {
132 pub use crate::generated::datafusion::*;
133 pub use datafusion_proto_common::common::proto_error;
134 pub use datafusion_proto_common::protobuf_common::{
135 ArrowFormat, ArrowOptions, ArrowType, AvroFormat, AvroOptions, CsvFormat,
136 DfSchema, EmptyMessage, Field, JoinSide, NdJsonFormat, ParquetFormat,
137 ScalarValue, Schema,
138 };
139 pub use datafusion_proto_common::{FromProtoError, ToProtoError};
140}
141
142#[cfg(doctest)]
143doc_comment::doctest!("../README.md", readme_example_test);