Skip to main content

buoyant_kernel/engine/plans/
mod.rs

1//! This module contains an implementation of the Engine trait that is
2//! backed by a PlanExecutor. The engine delegates handler operations
3//! (e.g. storage, JSON, parquet) to declarative plan execution rather than implementing
4//! each handler independently.
5//!
6//! This allows a PlanExecutor to become the single surface for connector optimizations,
7//! while still allowing kernel to use existing Engine trait APIs.
8//!
9//! ### Arrow Requirement:
10//! The PlanBasedEngine implementation assumes the use of ArrowEngineData during JSON parsing, so it
11//! is only compatible with `PlanExecutor`'s which return ArrowEngineData.
12
13use std::sync::Arc;
14
15pub mod json;
16pub mod parquet;
17pub mod storage;
18
19use json::PlanBasedJsonHandler;
20use parquet::PlanBasedParquetHandler;
21use storage::PlanBasedStorageHandler;
22
23use crate::engine::arrow_expression::ArrowEvaluationHandler;
24use crate::plans::PlanExecutor;
25use crate::{Engine, EvaluationHandler, JsonHandler, ParquetHandler, StorageHandler};
26
27/// An [`Engine`] that routes operations through a [`PlanExecutor`].
28///
29/// Storage, JSON file reads, and Parquet file reads are converted into
30/// [`Operation`](crate::plans::Operation)s and delegated to the plan executor.
31///
32/// Operations not yet implemented on the plan-execution path (e.g. `write_json_file`,
33/// `write_parquet_file`) are delegated to `fallback` when one is configured, and otherwise return
34/// an unsupported error. A fallback is optional because a connector may be unable to construct one:
35/// an FFI-backed engine resolves table paths itself, which fails for a path only the connector can
36/// resolve.
37pub struct PlanBasedEngine {
38    executor: Arc<dyn PlanExecutor>,
39    evaluation: Arc<dyn EvaluationHandler>,
40    storage: Arc<PlanBasedStorageHandler>,
41    json: Arc<PlanBasedJsonHandler>,
42    parquet: Arc<PlanBasedParquetHandler>,
43}
44
45impl PlanBasedEngine {
46    /// Construct a `PlanBasedEngine` backed by `plan_executor`, delegating operations not yet
47    /// implemented on the plan-execution path to `fallback` when it is provided.
48    ///
49    /// When `fallback` is `None`, those operations return an unsupported error. Evaluation uses the
50    /// fallback's [`EvaluationHandler`] when one is available and otherwise uses the Arrow
51    /// implementation.
52    pub fn new(fallback: Option<Arc<dyn Engine>>, plan_executor: Arc<dyn PlanExecutor>) -> Self {
53        let evaluation: Arc<dyn EvaluationHandler> = fallback.as_ref().map_or_else(
54            || Arc::new(ArrowEvaluationHandler) as Arc<dyn EvaluationHandler>,
55            |engine| engine.evaluation_handler(),
56        );
57        Self {
58            evaluation,
59            storage: Arc::new(PlanBasedStorageHandler::new(plan_executor.clone())),
60            json: Arc::new(PlanBasedJsonHandler::new(
61                plan_executor.clone(),
62                fallback.as_ref().map(|engine| engine.json_handler()),
63            )),
64            parquet: Arc::new(PlanBasedParquetHandler::new(
65                plan_executor.clone(),
66                fallback.as_ref().map(|engine| engine.parquet_handler()),
67            )),
68            executor: plan_executor,
69        }
70    }
71}
72
73impl Engine for PlanBasedEngine {
74    fn evaluation_handler(&self) -> Arc<dyn EvaluationHandler> {
75        self.evaluation.clone()
76    }
77
78    fn storage_handler(&self) -> Arc<dyn StorageHandler> {
79        self.storage.clone()
80    }
81
82    fn json_handler(&self) -> Arc<dyn JsonHandler> {
83        self.json.clone()
84    }
85
86    fn parquet_handler(&self) -> Arc<dyn ParquetHandler> {
87        self.parquet.clone()
88    }
89
90    fn plan_executor(&self) -> Option<Arc<dyn PlanExecutor>> {
91        Some(self.executor.clone())
92    }
93}
94
95#[cfg(test)]
96mod tests {
97    use std::sync::Arc;
98
99    use tempfile::tempdir;
100    use url::Url;
101
102    use super::PlanBasedEngine;
103    use crate::arrow::array::{Array, Int64Array, RecordBatch, StringArray};
104    use crate::engine::arrow_data::ArrowEngineData;
105    use crate::engine::arrow_expression::ArrowEvaluationHandler;
106    use crate::engine::sync::plan::SyncPlanExecutor;
107    use crate::engine::sync::SyncEngine;
108    use crate::engine_data::FilteredEngineData;
109    use crate::{Engine as _, EngineData, Error};
110
111    fn plan_engine(fallback: Option<Arc<dyn crate::Engine>>) -> PlanBasedEngine {
112        PlanBasedEngine::new(fallback, Arc::new(SyncPlanExecutor::default()))
113    }
114
115    #[test]
116    fn uses_fallback_evaluation_handler_when_available() {
117        let fallback: Arc<dyn crate::Engine> = Arc::new(SyncEngine::new());
118        let expected_evaluation_handler = fallback.evaluation_handler();
119        let engine = plan_engine(Some(fallback));
120
121        assert!(Arc::ptr_eq(
122            &engine.evaluation_handler(),
123            &expected_evaluation_handler
124        ));
125    }
126
127    #[test]
128    fn uses_arrow_evaluation_handler_without_fallback() {
129        let engine = plan_engine(None);
130
131        assert!(engine
132            .evaluation_handler()
133            .as_ref()
134            .any_ref()
135            .is::<ArrowEvaluationHandler>());
136    }
137
138    #[test]
139    fn unimplemented_writes_delegate_to_fallback() {
140        let dir = tempdir().unwrap();
141        let engine = plan_engine(Some(Arc::new(SyncEngine::new())));
142        let json_location = Url::from_file_path(dir.path().join("out.json")).unwrap();
143        let json_batch = RecordBatch::try_from_iter(vec![(
144            "value",
145            Arc::new(StringArray::from(vec!["one"])) as Arc<dyn Array>,
146        )])
147        .unwrap();
148        let json_data: Box<dyn EngineData> = Box::new(ArrowEngineData::new(json_batch));
149
150        engine
151            .json_handler()
152            .write_json_file(
153                &json_location,
154                Box::new(std::iter::once(Ok(
155                    FilteredEngineData::with_all_rows_selected(json_data),
156                ))),
157                false,
158            )
159            .unwrap();
160        assert_eq!(
161            std::fs::read_to_string(json_location.to_file_path().unwrap()).unwrap(),
162            "{\"value\":\"one\"}\n"
163        );
164
165        let parquet_location = Url::from_file_path(dir.path().join("out.parquet")).unwrap();
166        let parquet_batch = RecordBatch::try_from_iter(vec![(
167            "value",
168            Arc::new(Int64Array::from(vec![1])) as Arc<dyn Array>,
169        )])
170        .unwrap();
171        let parquet_data: Box<dyn EngineData> = Box::new(ArrowEngineData::new(parquet_batch));
172
173        engine
174            .parquet_handler()
175            .write_parquet_file(
176                parquet_location.clone(),
177                Box::new(std::iter::once(Ok(parquet_data))),
178            )
179            .unwrap();
180        assert!(
181            std::fs::metadata(parquet_location.to_file_path().unwrap())
182                .unwrap()
183                .len()
184                > 0
185        );
186    }
187
188    #[test]
189    fn unimplemented_writes_are_unsupported_without_fallback() {
190        let engine = plan_engine(None);
191        let location = Url::parse("memory:///table/out").unwrap();
192
193        let json_err = engine
194            .json_handler()
195            .write_json_file(&location, Box::new(std::iter::empty()), false)
196            .expect_err("no fallback is configured");
197        assert!(matches!(json_err, Error::Unsupported(_)));
198
199        let parquet_err = engine
200            .parquet_handler()
201            .write_parquet_file(location, Box::new(std::iter::empty()))
202            .expect_err("no fallback is configured");
203        assert!(matches!(parquet_err, Error::Unsupported(_)));
204    }
205}