Skip to main content

rustfs_s3select_api/
lib.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![recursion_limit = "256"]
16
17use datafusion::{
18    arrow::error::ArrowError,
19    common::{DataFusionError, SchemaError},
20    parquet::errors::ParquetError,
21    sql::sqlparser::parser::ParserError,
22};
23use std::{error::Error as StdError, fmt::Display};
24use thiserror::Error;
25
26mod csv_input;
27mod input_stream;
28mod metrics;
29pub mod object_store;
30pub mod query;
31pub mod server;
32mod storage_api;
33pub use csv_input::csv_input_requires_normalization;
34pub use metrics::{SelectInputMetrics, SelectInputMetricsSnapshot};
35pub use storage_api::SelectObjectSnapshot;
36
37#[cfg(test)]
38mod test;
39
40pub type QueryResult<T> = Result<T, QueryError>;
41pub(crate) use storage_api::crate_boundary::{
42    PrepareSelectObjectSnapshotError, SELECT_DEFAULT_READ_BUFFER_SIZE, SelectGetObjectReader, SelectObjectOptions,
43    SelectObjectSnapshotReadError, SelectStorageError, SelectStore, SnapshotConsistencyError, resolve_select_object_store_handle,
44    select_is_err_bucket_not_found, select_is_err_object_not_found, select_is_err_version_not_found,
45};
46
47#[derive(Debug, Error)]
48pub enum QueryError {
49    #[error("DataFusion error: {source}")]
50    Datafusion { source: Box<DataFusionError> },
51
52    #[error("This feature is not implemented: {err}")]
53    NotImplemented { err: String },
54
55    #[error("Multi-statement not allow, found num:{num}, sql:{sql}")]
56    MultiStatement { num: usize, sql: String },
57
58    #[error("Failed to build QueryDispatcher. err: {err}")]
59    BuildQueryDispatcher { err: String },
60
61    #[error("The query has been canceled")]
62    Cancel,
63
64    #[error("{source}")]
65    Parser {
66        #[from]
67        source: ParserError,
68    },
69
70    #[error("Udf not exists, name:{name}.")]
71    FunctionNotExists { name: String },
72
73    #[error("Udf already exists, name:{name}.")]
74    FunctionExists { name: String },
75
76    #[error("Store Error, e:{e}.")]
77    StoreError { e: String },
78}
79
80#[derive(Clone, Debug, Error, PartialEq, Eq)]
81pub enum SelectError {
82    #[error("The file is not in a supported compression format. Only GZIP and BZIP2 are supported.")]
83    InvalidCompressionFormat,
84
85    #[error("{compression} is not applicable to the queried object. Please correct the request and try again.")]
86    InvalidCompressionFormatForObject { compression: &'static str },
87
88    #[error("The data source type is not valid. Only CSV, JSON, and Parquet are supported.")]
89    InvalidDataSource,
90
91    #[error(
92        "Object decompression failed. Check that the object is properly compressed using the format specified in the request."
93    )]
94    TruncatedInput,
95
96    #[error("Scan range queries are not supported on this type of object.")]
97    UnsupportedScanRangeInput,
98
99    #[error("An error occurred while parsing the CSV file. Check the file and try again.")]
100    CsvParsingError,
101
102    #[error("An error occurred while parsing the JSON file. Check the file and try again.")]
103    JsonParsingError,
104
105    #[error("An error occurred while parsing the Parquet file. Check the file and try again.")]
106    ParquetParsingError,
107
108    #[error("The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB.")]
109    OverMaxRecordSize,
110
111    #[error("{message}")]
112    ParseSelectFailure { message: String },
113
114    #[error("The SQL expression is invalid.")]
115    InvalidQuery,
116
117    #[error("The SQL expression contains a data type that is not valid.")]
118    InvalidDataType,
119
120    #[error("An incorrect argument type was specified in a function call in the SQL expression.")]
121    IncorrectSqlFunctionArgumentType,
122
123    #[error("The data source path in the SQL expression is not supported.")]
124    DataSourcePathUnsupported,
125
126    #[error("Unsupported S3 Select SQL structure: {message}")]
127    UnsupportedSqlStructure { message: String },
128
129    #[error("We encountered an unsupported SQL operation.")]
130    UnsupportedSqlOperation,
131
132    #[error("A column name or a path provided does not exist in the SQL expression.")]
133    EvaluatorBindingDoesNotExist,
134
135    #[error("The field name matches to multiple fields in the file. Check the SQL expression and the file, and try again.")]
136    AmbiguousFieldName,
137
138    #[error("The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again.")]
139    InvalidScanRange,
140
141    #[error("S3 Select query concurrency limit reached")]
142    QueryConcurrencyLimit,
143
144    #[error("S3 Select query exceeded the {seconds}-second execution limit")]
145    QueryTimeout { seconds: u64 },
146
147    #[error("S3 Select query resource limit exceeded")]
148    ResourceExhausted,
149
150    #[error("The specified bucket does not exist.")]
151    BucketNotFound,
152
153    #[error("The specified key does not exist.")]
154    ObjectNotFound,
155
156    #[error("The query was canceled")]
157    Canceled,
158
159    #[error("An internal error occurred.")]
160    InternalError,
161}
162
163pub type S3SelectPolicyError = SelectError;
164
165const MAX_ERROR_SOURCE_DEPTH: usize = 16;
166
167impl QueryError {
168    fn source_error<T: StdError + 'static>(&self) -> Option<&T> {
169        let mut err: &(dyn StdError + 'static) = self;
170        for _ in 0..MAX_ERROR_SOURCE_DEPTH {
171            if let Some(source) = err.downcast_ref::<T>() {
172                return Some(source);
173            }
174            err = err.source()?;
175        }
176        None
177    }
178
179    pub fn is_snapshot_consistency_error(&self) -> bool {
180        self.source_error::<SnapshotConsistencyError>().is_some()
181    }
182
183    pub fn s3_select_policy_error(&self) -> Option<&S3SelectPolicyError> {
184        self.source_error()
185    }
186
187    pub fn select_error(&self) -> SelectError {
188        let mut err: &(dyn StdError + 'static) = match self {
189            Self::Datafusion { source } => source.as_ref(),
190            _ => self,
191        };
192        for _ in 0..MAX_ERROR_SOURCE_DEPTH {
193            if let Some(select_error) = classify_select_error_source(err) {
194                return select_error;
195            }
196            let Some(source) = err.source() else {
197                break;
198            };
199            err = source;
200        }
201
202        match self {
203            QueryError::NotImplemented { .. } => SelectError::UnsupportedSqlOperation,
204            QueryError::MultiStatement { .. } => SelectError::UnsupportedSqlStructure {
205                message: "multiple SQL statements are not supported".to_string(),
206            },
207            QueryError::BuildQueryDispatcher { .. } | QueryError::FunctionExists { .. } | QueryError::StoreError { .. } => {
208                SelectError::InternalError
209            }
210            QueryError::Cancel => SelectError::Canceled,
211            QueryError::FunctionNotExists { .. } => SelectError::InvalidQuery,
212            QueryError::Datafusion { .. } | QueryError::Parser { .. } => SelectError::InternalError,
213        }
214    }
215}
216
217fn classify_select_error_source(err: &(dyn StdError + 'static)) -> Option<SelectError> {
218    if let Some(error) = err.downcast_ref::<SelectError>() {
219        return Some(error.clone());
220    }
221    if let Some(error) = err.downcast_ref::<object_store::SelectObjectStoreError>() {
222        return Some(error.select_error());
223    }
224    if let Some(error) = err.downcast_ref::<datafusion::object_store::Error>() {
225        return match error {
226            datafusion::object_store::Error::NotFound { source, .. } => Some(
227                source
228                    .downcast_ref::<object_store::SelectObjectStoreError>()
229                    .map_or(SelectError::ObjectNotFound, object_store::SelectObjectStoreError::select_error),
230            ),
231            _ => None,
232        };
233    }
234    if let Some(error) = err.downcast_ref::<ParserError>() {
235        return Some(SelectError::ParseSelectFailure {
236            message: error.to_string(),
237        });
238    }
239    if let Some(error) = err.downcast_ref::<ArrowError>() {
240        return match error {
241            ArrowError::CsvError(_) => Some(SelectError::CsvParsingError),
242            ArrowError::JsonError(_) => Some(SelectError::JsonParsingError),
243            ArrowError::ParquetError(_) => Some(SelectError::ParquetParsingError),
244            ArrowError::CastError(_) | ArrowError::ParseError(_) => Some(SelectError::InvalidDataType),
245            ArrowError::MemoryError(_) => Some(SelectError::ResourceExhausted),
246            ArrowError::ExternalError(_) | ArrowError::IoError(_, _) => None,
247            _ => Some(SelectError::InternalError),
248        };
249    }
250    if let Some(error) = err.downcast_ref::<ParquetError>() {
251        return match error {
252            ParquetError::External(_) => None,
253            _ => Some(SelectError::ParquetParsingError),
254        };
255    }
256    if let Some(error) = err.downcast_ref::<SchemaError>() {
257        return Some(match error {
258            SchemaError::FieldNotFound { .. } => SelectError::EvaluatorBindingDoesNotExist,
259            SchemaError::AmbiguousReference { .. }
260            | SchemaError::DuplicateQualifiedField { .. }
261            | SchemaError::DuplicateUnqualifiedField { .. } => SelectError::AmbiguousFieldName,
262        });
263    }
264    if let Some(error) = err.downcast_ref::<DataFusionError>() {
265        return match error {
266            DataFusionError::NotImplemented(_) => Some(SelectError::UnsupportedSqlOperation),
267            DataFusionError::Plan(_) => Some(SelectError::InvalidQuery),
268            DataFusionError::ResourcesExhausted(_) => Some(SelectError::ResourceExhausted),
269            DataFusionError::Internal(_)
270            | DataFusionError::Execution(_)
271            | DataFusionError::Configuration(_)
272            | DataFusionError::Substrait(_)
273            | DataFusionError::Ffi(_) => Some(SelectError::InternalError),
274            DataFusionError::ArrowError(_, _)
275            | DataFusionError::ParquetError(_)
276            | DataFusionError::ObjectStore(_)
277            | DataFusionError::IoError(_)
278            | DataFusionError::SQL(_, _)
279            | DataFusionError::SchemaError(_, _)
280            | DataFusionError::ExecutionJoin(_)
281            | DataFusionError::External(_)
282            | DataFusionError::Context(_, _)
283            | DataFusionError::Diagnostic(_, _)
284            | DataFusionError::Collection(_)
285            | DataFusionError::Shared(_) => None,
286        };
287    }
288    None
289}
290
291impl From<SelectError> for QueryError {
292    fn from(value: SelectError) -> Self {
293        Self::Datafusion {
294            source: Box::new(DataFusionError::External(Box::new(value))),
295        }
296    }
297}
298
299impl From<DataFusionError> for QueryError {
300    fn from(value: DataFusionError) -> Self {
301        match value {
302            DataFusionError::External(e) => match e.downcast::<QueryError>() {
303                Ok(query_error) => *query_error,
304                Err(e) => Self::Datafusion {
305                    source: Box::new(DataFusionError::External(e)),
306                },
307            },
308            v => Self::Datafusion { source: Box::new(v) },
309        }
310    }
311}
312
313#[derive(Debug, Clone, PartialEq, Eq)]
314pub struct ResolvedTable {
315    // path
316    table: String,
317}
318
319impl ResolvedTable {
320    pub fn table(&self) -> &str {
321        &self.table
322    }
323}
324
325impl Display for ResolvedTable {
326    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
327        let Self { table } = self;
328        write!(f, "{table}")
329    }
330}
331
332#[cfg(test)]
333mod tests {
334    use super::*;
335    use datafusion::common::DataFusionError;
336    use datafusion::sql::sqlparser::parser::ParserError;
337
338    #[test]
339    fn test_query_error_display() {
340        let err = QueryError::NotImplemented {
341            err: "feature X".to_string(),
342        };
343        assert_eq!(err.to_string(), "This feature is not implemented: feature X");
344
345        let err = QueryError::MultiStatement {
346            num: 2,
347            sql: "SELECT 1; SELECT 2;".to_string(),
348        };
349        assert_eq!(err.to_string(), "Multi-statement not allow, found num:2, sql:SELECT 1; SELECT 2;");
350
351        let err = SelectError::UnsupportedSqlStructure {
352            message: "JOIN is not supported".to_string(),
353        };
354        assert_eq!(err.to_string(), "Unsupported S3 Select SQL structure: JOIN is not supported");
355
356        let err = QueryError::Cancel;
357        assert_eq!(err.to_string(), "The query has been canceled");
358
359        assert_eq!(
360            SelectError::QueryConcurrencyLimit.to_string(),
361            "S3 Select query concurrency limit reached"
362        );
363        assert_eq!(
364            SelectError::QueryTimeout { seconds: 300 }.to_string(),
365            "S3 Select query exceeded the 300-second execution limit"
366        );
367
368        let err = QueryError::FunctionNotExists {
369            name: "my_func".to_string(),
370        };
371        assert_eq!(err.to_string(), "Udf not exists, name:my_func.");
372
373        let err = QueryError::StoreError {
374            e: "connection failed".to_string(),
375        };
376        assert_eq!(err.to_string(), "Store Error, e:connection failed.");
377    }
378
379    #[test]
380    fn test_query_error_from_datafusion_error() {
381        let df_error = DataFusionError::Plan("invalid plan".to_string());
382        let query_error: QueryError = df_error.into();
383
384        match query_error {
385            QueryError::Datafusion { source, .. } => {
386                assert!(source.to_string().contains("invalid plan"));
387            }
388            _ => panic!("Expected Datafusion error"),
389        }
390    }
391
392    #[test]
393    fn query_error_variants_remain_source_compatible() {
394        fn exhaustive_match(err: QueryError) {
395            match err {
396                QueryError::Datafusion { .. }
397                | QueryError::NotImplemented { .. }
398                | QueryError::MultiStatement { .. }
399                | QueryError::BuildQueryDispatcher { .. }
400                | QueryError::Cancel
401                | QueryError::Parser { .. }
402                | QueryError::FunctionNotExists { .. }
403                | QueryError::FunctionExists { .. }
404                | QueryError::StoreError { .. } => {}
405            }
406        }
407
408        exhaustive_match(QueryError::Cancel);
409    }
410
411    #[test]
412    fn policy_error_is_recoverable_from_query_error() {
413        let err: QueryError = SelectError::QueryTimeout { seconds: 300 }.into();
414
415        assert!(matches!(err.s3_select_policy_error(), Some(SelectError::QueryTimeout { seconds: 300 })));
416    }
417
418    #[test]
419    fn query_error_classifies_data_errors_without_display_matching() {
420        let cases = [
421            (
422                DataFusionError::ArrowError(Box::new(ArrowError::CsvError("private csv detail".to_string())), None),
423                SelectError::CsvParsingError,
424            ),
425            (
426                DataFusionError::ArrowError(Box::new(ArrowError::JsonError("private json detail".to_string())), None),
427                SelectError::JsonParsingError,
428            ),
429            (
430                DataFusionError::ParquetError(Box::new(ParquetError::General("private parquet detail".to_string()))),
431                SelectError::ParquetParsingError,
432            ),
433            (
434                DataFusionError::External(Box::new(SelectError::TruncatedInput)),
435                SelectError::TruncatedInput,
436            ),
437            (
438                DataFusionError::ArrowError(
439                    Box::new(ArrowError::InvalidArgumentError("private implementation detail".to_string())),
440                    None,
441                ),
442                SelectError::InternalError,
443            ),
444            (
445                DataFusionError::ArrowError(Box::new(ArrowError::CastError("invalid cast".to_string())), None),
446                SelectError::InvalidDataType,
447            ),
448            (
449                DataFusionError::ArrowError(Box::new(ArrowError::MemoryError("query memory limit".to_string())), None),
450                SelectError::ResourceExhausted,
451            ),
452            (
453                DataFusionError::Execution("private execution detail".to_string()),
454                SelectError::InternalError,
455            ),
456            (DataFusionError::Plan("invalid expression".to_string()), SelectError::InvalidQuery),
457            (
458                DataFusionError::NotImplemented("unsupported expression".to_string()),
459                SelectError::UnsupportedSqlOperation,
460            ),
461            (
462                DataFusionError::SchemaError(
463                    Box::new(SchemaError::FieldNotFound {
464                        field: Box::new(datafusion::common::Column::from_name("missing")),
465                        valid_fields: Vec::new(),
466                    }),
467                    Box::new(None),
468                ),
469                SelectError::EvaluatorBindingDoesNotExist,
470            ),
471            (
472                DataFusionError::SchemaError(
473                    Box::new(SchemaError::AmbiguousReference {
474                        field: Box::new(datafusion::common::Column::from_name("duplicate")),
475                    }),
476                    Box::new(None),
477                ),
478                SelectError::AmbiguousFieldName,
479            ),
480        ];
481
482        for (source, expected) in cases {
483            let error = QueryError::from(source);
484            assert_eq!(error.select_error(), expected, "wrong classification for {error:?}");
485        }
486    }
487
488    #[test]
489    fn query_error_preserves_typed_object_store_classification() {
490        let bucket_error = QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::NotFound {
491            path: "private-bucket/private-object".to_string(),
492            source: Box::new(object_store::SelectObjectStoreError::BucketNotFound {
493                source: SelectStorageError::BucketNotFound("private-bucket".to_string()),
494            }),
495        })));
496        let object_error = QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::NotFound {
497            path: "private-bucket/private-object".to_string(),
498            source: Box::new(object_store::SelectObjectStoreError::ObjectNotFound {
499                source: SelectStorageError::ObjectNotFound("private-bucket".to_string(), "private-object".to_string()),
500            }),
501        })));
502        let scan_range_error =
503            QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::Generic {
504                store: "test",
505                source: Box::new(object_store::SelectObjectStoreError::InvalidScanRange),
506            })));
507        let storage_error = QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::Generic {
508            store: "test",
509            source: Box::new(object_store::SelectObjectStoreError::Storage {
510                source: SelectStorageError::LessData,
511            }),
512        })));
513
514        assert_eq!(bucket_error.select_error(), SelectError::BucketNotFound);
515        assert_eq!(object_error.select_error(), SelectError::ObjectNotFound);
516        assert_eq!(scan_range_error.select_error(), SelectError::InvalidScanRange);
517        assert_eq!(storage_error.select_error(), SelectError::InternalError);
518    }
519
520    #[test]
521    fn select_error_source_traversal_stops_at_the_depth_bound() {
522        #[derive(Debug)]
523        struct CyclicError;
524
525        impl std::fmt::Display for CyclicError {
526            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
527                f.write_str("cyclic error")
528            }
529        }
530
531        impl StdError for CyclicError {
532            fn source(&self) -> Option<&(dyn StdError + 'static)> {
533                Some(self)
534            }
535        }
536
537        let error = QueryError::from(DataFusionError::External(Box::new(CyclicError)));
538        assert_eq!(error.select_error(), SelectError::InternalError);
539    }
540
541    #[test]
542    fn snapshot_consistency_error_is_recoverable_without_string_matching() {
543        let err = QueryError::Datafusion {
544            source: Box::new(DataFusionError::External(Box::new(SelectObjectSnapshotReadError::Consistency(
545                SnapshotConsistencyError::LockLost,
546            )))),
547        };
548
549        assert!(err.is_snapshot_consistency_error());
550    }
551
552    #[test]
553    fn test_query_error_from_parser_error() {
554        let parser_error = ParserError::ParserError("syntax error".to_string());
555        let query_error = QueryError::Parser { source: parser_error };
556
557        assert!(query_error.to_string().contains("syntax error"));
558    }
559
560    #[test]
561    fn test_resolved_table() {
562        let table = ResolvedTable {
563            table: "my_table".to_string(),
564        };
565
566        assert_eq!(table.table(), "my_table");
567        assert_eq!(table.to_string(), "my_table");
568    }
569
570    #[test]
571    fn test_resolved_table_clone_and_eq() {
572        let table1 = ResolvedTable {
573            table: "table1".to_string(),
574        };
575        let table2 = table1.clone();
576        let table3 = ResolvedTable {
577            table: "table2".to_string(),
578        };
579
580        assert_eq!(table1, table2);
581        assert_ne!(table1, table3);
582    }
583}