datafusion_proto/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18#![doc(
19    html_logo_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg",
20    html_favicon_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg"
21)]
22#![cfg_attr(docsrs, feature(doc_cfg))]
23// Make sure fast / cheap clones on Arc are explicit:
24// https://github.com/apache/datafusion/issues/11143
25#![deny(clippy::clone_on_ref_ptr)]
26
27//! Serialize / Deserialize DataFusion Plans to bytes
28//!
29//! This crate provides support for serializing and deserializing the
30//! following structures to and from bytes:
31//!
32//! 1. [`LogicalPlan`]'s (including [`Expr`]),
33//! 2. [`ExecutionPlan`]s (including [`PhysicalExpr`])
34//!
35//! [`LogicalPlan`]: datafusion_expr::LogicalPlan
36//! [`Expr`]: datafusion_expr::Expr
37//! [`ExecutionPlan`]: datafusion::physical_plan::ExecutionPlan
38//! [`PhysicalExpr`]: datafusion::physical_expr::PhysicalExpr
39//!
40//! Internally, this crate is implemented by converting the plans to [protocol
41//! buffers] using [prost].
42//!
43//! [protocol buffers]: https://developers.google.com/protocol-buffers
44//! [prost]: https://docs.rs/prost/latest/prost/
45//!
46//! # Version Compatibility
47//!
48//! The serialized form are not guaranteed to be compatible across
49//! DataFusion versions. A plan serialized with one version of DataFusion
50//! may not be able to deserialized with a different version.
51//!
52//! # See Also
53//!
54//! The binary format created by this crate supports the full range of DataFusion
55//! plans, but is DataFusion specific. See [datafusion-substrait] for a crate
56//! which can encode many DataFusion plans using the [substrait.io] standard.
57//!
58//! [datafusion-substrait]: https://docs.rs/datafusion-substrait/latest/datafusion_substrait
59//! [substrait.io]: https://substrait.io
60//!
61//! # Example: Serializing [`Expr`]s
62//! ```
63//! # use datafusion_common::Result;
64//! # use datafusion_expr::{col, lit, Expr};
65//! # use datafusion_proto::bytes::Serializeable;
66//! # fn main() -> Result<()>{
67//!  // Create a new `Expr` a < 32
68//!  let expr = col("a").lt(lit(5i32));
69//!
70//!  // Convert it to bytes (for sending over the network, etc.)
71//!  let bytes = expr.to_bytes()?;
72//!
73//!  // Decode bytes from somewhere (over network, etc.) back to Expr
74//!  let decoded_expr = Expr::from_bytes(&bytes)?;
75//!  assert_eq!(expr, decoded_expr);
76//! # Ok(())
77//! # }
78//! ```
79//!
80//! # Example: Serializing [`LogicalPlan`]s
81//! ```
82//! # use datafusion::prelude::*;
83//! # use datafusion_common::Result;
84//! # use datafusion_proto::bytes::{logical_plan_from_bytes, logical_plan_to_bytes};
85//! # #[tokio::main]
86//! # async fn main() -> Result<()>{
87//!  // Create a plan that scans table 't'
88//!  let ctx = SessionContext::new();
89//!  ctx.register_csv("t1", "tests/testdata/test.csv", CsvReadOptions::default()).await?;
90//!  let plan = ctx.table("t1").await?.into_optimized_plan()?;
91//!
92//!  // Convert the plan into bytes (for sending over the network, etc.)
93//!  let bytes = logical_plan_to_bytes(&plan)?;
94//!
95//!  // Decode bytes from somewhere (over network, etc.) back to LogicalPlan
96//!  let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?;
97//!  assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip));
98//! # Ok(())
99//! # }
100//! ```
101//! # Example: Serializing [`ExecutionPlan`]s
102//!
103//! ```
104//! # use datafusion::prelude::*;
105//! # use datafusion_common::Result;
106//! # use datafusion_proto::bytes::{physical_plan_from_bytes,physical_plan_to_bytes};
107//! # #[tokio::main]
108//! # async fn main() -> Result<()>{
109//!  // Create a plan that scans table 't'
110//!  let ctx = SessionContext::new();
111//!  ctx.register_csv("t1", "tests/testdata/test.csv", CsvReadOptions::default()).await?;
112//!  let physical_plan = ctx.table("t1").await?.create_physical_plan().await?;
113//!
114//!  // Convert the plan into bytes (for sending over the network, etc.)
115//!  let bytes = physical_plan_to_bytes(physical_plan.clone())?;
116//!
117//!  // Decode bytes from somewhere (over network, etc.) back to ExecutionPlan
118//!  let physical_round_trip = physical_plan_from_bytes(&bytes, &ctx)?;
119//!  assert_eq!(format!("{:?}", physical_plan), format!("{:?}", physical_round_trip));
120//! # Ok(())
121//! # }
122//! ```
123pub mod bytes;
124pub mod common;
125pub mod generated;
126pub mod logical_plan;
127pub mod physical_plan;
128
129pub mod protobuf {
130    pub use crate::generated::datafusion::*;
131    pub use datafusion_proto_common::common::proto_error;
132    pub use datafusion_proto_common::protobuf_common::{
133        ArrowFormat, ArrowOptions, ArrowType, AvroFormat, AvroOptions, CsvFormat,
134        DfSchema, EmptyMessage, Field, JoinSide, NdJsonFormat, ParquetFormat,
135        ScalarValue, Schema,
136    };
137    pub use datafusion_proto_common::{FromProtoError, ToProtoError};
138}
139
140#[cfg(doctest)]
141doc_comment::doctest!("../README.md", readme_example_test);
datafusion_proto/lib.rs

datafusion_proto/
lib.rs
