1#![recursion_limit = "256"]
16
17use datafusion::{
18 arrow::error::ArrowError,
19 common::{DataFusionError, SchemaError},
20 parquet::errors::ParquetError,
21 sql::sqlparser::parser::ParserError,
22};
23use std::{error::Error as StdError, fmt::Display};
24use thiserror::Error;
25
26mod csv_input;
27mod input_stream;
28mod metrics;
29pub mod object_store;
30pub mod query;
31pub mod server;
32mod storage_api;
33pub use csv_input::csv_input_requires_normalization;
34pub use metrics::{SelectInputMetrics, SelectInputMetricsSnapshot};
35pub use storage_api::SelectObjectSnapshot;
36
37#[cfg(test)]
38mod test;
39
40pub type QueryResult<T> = Result<T, QueryError>;
41pub(crate) use storage_api::crate_boundary::{
42 PrepareSelectObjectSnapshotError, SELECT_DEFAULT_READ_BUFFER_SIZE, SelectGetObjectReader, SelectObjectOptions,
43 SelectObjectSnapshotReadError, SelectStorageError, SelectStore, SnapshotConsistencyError, resolve_select_object_store_handle,
44 select_is_err_bucket_not_found, select_is_err_object_not_found, select_is_err_version_not_found,
45};
46
47#[derive(Debug, Error)]
48pub enum QueryError {
49 #[error("DataFusion error: {source}")]
50 Datafusion { source: Box<DataFusionError> },
51
52 #[error("This feature is not implemented: {err}")]
53 NotImplemented { err: String },
54
55 #[error("Multi-statement not allow, found num:{num}, sql:{sql}")]
56 MultiStatement { num: usize, sql: String },
57
58 #[error("Failed to build QueryDispatcher. err: {err}")]
59 BuildQueryDispatcher { err: String },
60
61 #[error("The query has been canceled")]
62 Cancel,
63
64 #[error("{source}")]
65 Parser {
66 #[from]
67 source: ParserError,
68 },
69
70 #[error("Udf not exists, name:{name}.")]
71 FunctionNotExists { name: String },
72
73 #[error("Udf already exists, name:{name}.")]
74 FunctionExists { name: String },
75
76 #[error("Store Error, e:{e}.")]
77 StoreError { e: String },
78}
79
80#[derive(Clone, Debug, Error, PartialEq, Eq)]
81pub enum SelectError {
82 #[error("The file is not in a supported compression format. Only GZIP and BZIP2 are supported.")]
83 InvalidCompressionFormat,
84
85 #[error("{compression} is not applicable to the queried object. Please correct the request and try again.")]
86 InvalidCompressionFormatForObject { compression: &'static str },
87
88 #[error("The data source type is not valid. Only CSV, JSON, and Parquet are supported.")]
89 InvalidDataSource,
90
91 #[error(
92 "Object decompression failed. Check that the object is properly compressed using the format specified in the request."
93 )]
94 TruncatedInput,
95
96 #[error("Scan range queries are not supported on this type of object.")]
97 UnsupportedScanRangeInput,
98
99 #[error("An error occurred while parsing the CSV file. Check the file and try again.")]
100 CsvParsingError,
101
102 #[error("An error occurred while parsing the JSON file. Check the file and try again.")]
103 JsonParsingError,
104
105 #[error("An error occurred while parsing the Parquet file. Check the file and try again.")]
106 ParquetParsingError,
107
108 #[error("The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB.")]
109 OverMaxRecordSize,
110
111 #[error("{message}")]
112 ParseSelectFailure { message: String },
113
114 #[error("The SQL expression is invalid.")]
115 InvalidQuery,
116
117 #[error("The SQL expression contains a data type that is not valid.")]
118 InvalidDataType,
119
120 #[error("An incorrect argument type was specified in a function call in the SQL expression.")]
121 IncorrectSqlFunctionArgumentType,
122
123 #[error("The data source path in the SQL expression is not supported.")]
124 DataSourcePathUnsupported,
125
126 #[error("Unsupported S3 Select SQL structure: {message}")]
127 UnsupportedSqlStructure { message: String },
128
129 #[error("We encountered an unsupported SQL operation.")]
130 UnsupportedSqlOperation,
131
132 #[error("A column name or a path provided does not exist in the SQL expression.")]
133 EvaluatorBindingDoesNotExist,
134
135 #[error("The field name matches to multiple fields in the file. Check the SQL expression and the file, and try again.")]
136 AmbiguousFieldName,
137
138 #[error("The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again.")]
139 InvalidScanRange,
140
141 #[error("S3 Select query concurrency limit reached")]
142 QueryConcurrencyLimit,
143
144 #[error("S3 Select query exceeded the {seconds}-second execution limit")]
145 QueryTimeout { seconds: u64 },
146
147 #[error("S3 Select query resource limit exceeded")]
148 ResourceExhausted,
149
150 #[error("The specified bucket does not exist.")]
151 BucketNotFound,
152
153 #[error("The specified key does not exist.")]
154 ObjectNotFound,
155
156 #[error("The query was canceled")]
157 Canceled,
158
159 #[error("An internal error occurred.")]
160 InternalError,
161}
162
163pub type S3SelectPolicyError = SelectError;
164
165const MAX_ERROR_SOURCE_DEPTH: usize = 16;
166
167impl QueryError {
168 fn source_error<T: StdError + 'static>(&self) -> Option<&T> {
169 let mut err: &(dyn StdError + 'static) = self;
170 for _ in 0..MAX_ERROR_SOURCE_DEPTH {
171 if let Some(source) = err.downcast_ref::<T>() {
172 return Some(source);
173 }
174 err = err.source()?;
175 }
176 None
177 }
178
179 pub fn is_snapshot_consistency_error(&self) -> bool {
180 self.source_error::<SnapshotConsistencyError>().is_some()
181 }
182
183 pub fn s3_select_policy_error(&self) -> Option<&S3SelectPolicyError> {
184 self.source_error()
185 }
186
187 pub fn select_error(&self) -> SelectError {
188 let mut err: &(dyn StdError + 'static) = match self {
189 Self::Datafusion { source } => source.as_ref(),
190 _ => self,
191 };
192 for _ in 0..MAX_ERROR_SOURCE_DEPTH {
193 if let Some(select_error) = classify_select_error_source(err) {
194 return select_error;
195 }
196 let Some(source) = err.source() else {
197 break;
198 };
199 err = source;
200 }
201
202 match self {
203 QueryError::NotImplemented { .. } => SelectError::UnsupportedSqlOperation,
204 QueryError::MultiStatement { .. } => SelectError::UnsupportedSqlStructure {
205 message: "multiple SQL statements are not supported".to_string(),
206 },
207 QueryError::BuildQueryDispatcher { .. } | QueryError::FunctionExists { .. } | QueryError::StoreError { .. } => {
208 SelectError::InternalError
209 }
210 QueryError::Cancel => SelectError::Canceled,
211 QueryError::FunctionNotExists { .. } => SelectError::InvalidQuery,
212 QueryError::Datafusion { .. } | QueryError::Parser { .. } => SelectError::InternalError,
213 }
214 }
215}
216
217fn classify_select_error_source(err: &(dyn StdError + 'static)) -> Option<SelectError> {
218 if let Some(error) = err.downcast_ref::<SelectError>() {
219 return Some(error.clone());
220 }
221 if let Some(error) = err.downcast_ref::<object_store::SelectObjectStoreError>() {
222 return Some(error.select_error());
223 }
224 if let Some(error) = err.downcast_ref::<datafusion::object_store::Error>() {
225 return match error {
226 datafusion::object_store::Error::NotFound { source, .. } => Some(
227 source
228 .downcast_ref::<object_store::SelectObjectStoreError>()
229 .map_or(SelectError::ObjectNotFound, object_store::SelectObjectStoreError::select_error),
230 ),
231 _ => None,
232 };
233 }
234 if let Some(error) = err.downcast_ref::<ParserError>() {
235 return Some(SelectError::ParseSelectFailure {
236 message: error.to_string(),
237 });
238 }
239 if let Some(error) = err.downcast_ref::<ArrowError>() {
240 return match error {
241 ArrowError::CsvError(_) => Some(SelectError::CsvParsingError),
242 ArrowError::JsonError(_) => Some(SelectError::JsonParsingError),
243 ArrowError::ParquetError(_) => Some(SelectError::ParquetParsingError),
244 ArrowError::CastError(_) | ArrowError::ParseError(_) => Some(SelectError::InvalidDataType),
245 ArrowError::MemoryError(_) => Some(SelectError::ResourceExhausted),
246 ArrowError::ExternalError(_) | ArrowError::IoError(_, _) => None,
247 _ => Some(SelectError::InternalError),
248 };
249 }
250 if let Some(error) = err.downcast_ref::<ParquetError>() {
251 return match error {
252 ParquetError::External(_) => None,
253 _ => Some(SelectError::ParquetParsingError),
254 };
255 }
256 if let Some(error) = err.downcast_ref::<SchemaError>() {
257 return Some(match error {
258 SchemaError::FieldNotFound { .. } => SelectError::EvaluatorBindingDoesNotExist,
259 SchemaError::AmbiguousReference { .. }
260 | SchemaError::DuplicateQualifiedField { .. }
261 | SchemaError::DuplicateUnqualifiedField { .. } => SelectError::AmbiguousFieldName,
262 });
263 }
264 if let Some(error) = err.downcast_ref::<DataFusionError>() {
265 return match error {
266 DataFusionError::NotImplemented(_) => Some(SelectError::UnsupportedSqlOperation),
267 DataFusionError::Plan(_) => Some(SelectError::InvalidQuery),
268 DataFusionError::ResourcesExhausted(_) => Some(SelectError::ResourceExhausted),
269 DataFusionError::Internal(_)
270 | DataFusionError::Execution(_)
271 | DataFusionError::Configuration(_)
272 | DataFusionError::Substrait(_)
273 | DataFusionError::Ffi(_) => Some(SelectError::InternalError),
274 DataFusionError::ArrowError(_, _)
275 | DataFusionError::ParquetError(_)
276 | DataFusionError::ObjectStore(_)
277 | DataFusionError::IoError(_)
278 | DataFusionError::SQL(_, _)
279 | DataFusionError::SchemaError(_, _)
280 | DataFusionError::ExecutionJoin(_)
281 | DataFusionError::External(_)
282 | DataFusionError::Context(_, _)
283 | DataFusionError::Diagnostic(_, _)
284 | DataFusionError::Collection(_)
285 | DataFusionError::Shared(_) => None,
286 };
287 }
288 None
289}
290
291impl From<SelectError> for QueryError {
292 fn from(value: SelectError) -> Self {
293 Self::Datafusion {
294 source: Box::new(DataFusionError::External(Box::new(value))),
295 }
296 }
297}
298
299impl From<DataFusionError> for QueryError {
300 fn from(value: DataFusionError) -> Self {
301 match value {
302 DataFusionError::External(e) => match e.downcast::<QueryError>() {
303 Ok(query_error) => *query_error,
304 Err(e) => Self::Datafusion {
305 source: Box::new(DataFusionError::External(e)),
306 },
307 },
308 v => Self::Datafusion { source: Box::new(v) },
309 }
310 }
311}
312
313#[derive(Debug, Clone, PartialEq, Eq)]
314pub struct ResolvedTable {
315 table: String,
317}
318
319impl ResolvedTable {
320 pub fn table(&self) -> &str {
321 &self.table
322 }
323}
324
325impl Display for ResolvedTable {
326 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
327 let Self { table } = self;
328 write!(f, "{table}")
329 }
330}
331
332#[cfg(test)]
333mod tests {
334 use super::*;
335 use datafusion::common::DataFusionError;
336 use datafusion::sql::sqlparser::parser::ParserError;
337
338 #[test]
339 fn test_query_error_display() {
340 let err = QueryError::NotImplemented {
341 err: "feature X".to_string(),
342 };
343 assert_eq!(err.to_string(), "This feature is not implemented: feature X");
344
345 let err = QueryError::MultiStatement {
346 num: 2,
347 sql: "SELECT 1; SELECT 2;".to_string(),
348 };
349 assert_eq!(err.to_string(), "Multi-statement not allow, found num:2, sql:SELECT 1; SELECT 2;");
350
351 let err = SelectError::UnsupportedSqlStructure {
352 message: "JOIN is not supported".to_string(),
353 };
354 assert_eq!(err.to_string(), "Unsupported S3 Select SQL structure: JOIN is not supported");
355
356 let err = QueryError::Cancel;
357 assert_eq!(err.to_string(), "The query has been canceled");
358
359 assert_eq!(
360 SelectError::QueryConcurrencyLimit.to_string(),
361 "S3 Select query concurrency limit reached"
362 );
363 assert_eq!(
364 SelectError::QueryTimeout { seconds: 300 }.to_string(),
365 "S3 Select query exceeded the 300-second execution limit"
366 );
367
368 let err = QueryError::FunctionNotExists {
369 name: "my_func".to_string(),
370 };
371 assert_eq!(err.to_string(), "Udf not exists, name:my_func.");
372
373 let err = QueryError::StoreError {
374 e: "connection failed".to_string(),
375 };
376 assert_eq!(err.to_string(), "Store Error, e:connection failed.");
377 }
378
379 #[test]
380 fn test_query_error_from_datafusion_error() {
381 let df_error = DataFusionError::Plan("invalid plan".to_string());
382 let query_error: QueryError = df_error.into();
383
384 match query_error {
385 QueryError::Datafusion { source, .. } => {
386 assert!(source.to_string().contains("invalid plan"));
387 }
388 _ => panic!("Expected Datafusion error"),
389 }
390 }
391
392 #[test]
393 fn query_error_variants_remain_source_compatible() {
394 fn exhaustive_match(err: QueryError) {
395 match err {
396 QueryError::Datafusion { .. }
397 | QueryError::NotImplemented { .. }
398 | QueryError::MultiStatement { .. }
399 | QueryError::BuildQueryDispatcher { .. }
400 | QueryError::Cancel
401 | QueryError::Parser { .. }
402 | QueryError::FunctionNotExists { .. }
403 | QueryError::FunctionExists { .. }
404 | QueryError::StoreError { .. } => {}
405 }
406 }
407
408 exhaustive_match(QueryError::Cancel);
409 }
410
411 #[test]
412 fn policy_error_is_recoverable_from_query_error() {
413 let err: QueryError = SelectError::QueryTimeout { seconds: 300 }.into();
414
415 assert!(matches!(err.s3_select_policy_error(), Some(SelectError::QueryTimeout { seconds: 300 })));
416 }
417
418 #[test]
419 fn query_error_classifies_data_errors_without_display_matching() {
420 let cases = [
421 (
422 DataFusionError::ArrowError(Box::new(ArrowError::CsvError("private csv detail".to_string())), None),
423 SelectError::CsvParsingError,
424 ),
425 (
426 DataFusionError::ArrowError(Box::new(ArrowError::JsonError("private json detail".to_string())), None),
427 SelectError::JsonParsingError,
428 ),
429 (
430 DataFusionError::ParquetError(Box::new(ParquetError::General("private parquet detail".to_string()))),
431 SelectError::ParquetParsingError,
432 ),
433 (
434 DataFusionError::External(Box::new(SelectError::TruncatedInput)),
435 SelectError::TruncatedInput,
436 ),
437 (
438 DataFusionError::ArrowError(
439 Box::new(ArrowError::InvalidArgumentError("private implementation detail".to_string())),
440 None,
441 ),
442 SelectError::InternalError,
443 ),
444 (
445 DataFusionError::ArrowError(Box::new(ArrowError::CastError("invalid cast".to_string())), None),
446 SelectError::InvalidDataType,
447 ),
448 (
449 DataFusionError::ArrowError(Box::new(ArrowError::MemoryError("query memory limit".to_string())), None),
450 SelectError::ResourceExhausted,
451 ),
452 (
453 DataFusionError::Execution("private execution detail".to_string()),
454 SelectError::InternalError,
455 ),
456 (DataFusionError::Plan("invalid expression".to_string()), SelectError::InvalidQuery),
457 (
458 DataFusionError::NotImplemented("unsupported expression".to_string()),
459 SelectError::UnsupportedSqlOperation,
460 ),
461 (
462 DataFusionError::SchemaError(
463 Box::new(SchemaError::FieldNotFound {
464 field: Box::new(datafusion::common::Column::from_name("missing")),
465 valid_fields: Vec::new(),
466 }),
467 Box::new(None),
468 ),
469 SelectError::EvaluatorBindingDoesNotExist,
470 ),
471 (
472 DataFusionError::SchemaError(
473 Box::new(SchemaError::AmbiguousReference {
474 field: Box::new(datafusion::common::Column::from_name("duplicate")),
475 }),
476 Box::new(None),
477 ),
478 SelectError::AmbiguousFieldName,
479 ),
480 ];
481
482 for (source, expected) in cases {
483 let error = QueryError::from(source);
484 assert_eq!(error.select_error(), expected, "wrong classification for {error:?}");
485 }
486 }
487
488 #[test]
489 fn query_error_preserves_typed_object_store_classification() {
490 let bucket_error = QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::NotFound {
491 path: "private-bucket/private-object".to_string(),
492 source: Box::new(object_store::SelectObjectStoreError::BucketNotFound {
493 source: SelectStorageError::BucketNotFound("private-bucket".to_string()),
494 }),
495 })));
496 let object_error = QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::NotFound {
497 path: "private-bucket/private-object".to_string(),
498 source: Box::new(object_store::SelectObjectStoreError::ObjectNotFound {
499 source: SelectStorageError::ObjectNotFound("private-bucket".to_string(), "private-object".to_string()),
500 }),
501 })));
502 let scan_range_error =
503 QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::Generic {
504 store: "test",
505 source: Box::new(object_store::SelectObjectStoreError::InvalidScanRange),
506 })));
507 let storage_error = QueryError::from(DataFusionError::ObjectStore(Box::new(datafusion::object_store::Error::Generic {
508 store: "test",
509 source: Box::new(object_store::SelectObjectStoreError::Storage {
510 source: SelectStorageError::LessData,
511 }),
512 })));
513
514 assert_eq!(bucket_error.select_error(), SelectError::BucketNotFound);
515 assert_eq!(object_error.select_error(), SelectError::ObjectNotFound);
516 assert_eq!(scan_range_error.select_error(), SelectError::InvalidScanRange);
517 assert_eq!(storage_error.select_error(), SelectError::InternalError);
518 }
519
520 #[test]
521 fn select_error_source_traversal_stops_at_the_depth_bound() {
522 #[derive(Debug)]
523 struct CyclicError;
524
525 impl std::fmt::Display for CyclicError {
526 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
527 f.write_str("cyclic error")
528 }
529 }
530
531 impl StdError for CyclicError {
532 fn source(&self) -> Option<&(dyn StdError + 'static)> {
533 Some(self)
534 }
535 }
536
537 let error = QueryError::from(DataFusionError::External(Box::new(CyclicError)));
538 assert_eq!(error.select_error(), SelectError::InternalError);
539 }
540
541 #[test]
542 fn snapshot_consistency_error_is_recoverable_without_string_matching() {
543 let err = QueryError::Datafusion {
544 source: Box::new(DataFusionError::External(Box::new(SelectObjectSnapshotReadError::Consistency(
545 SnapshotConsistencyError::LockLost,
546 )))),
547 };
548
549 assert!(err.is_snapshot_consistency_error());
550 }
551
552 #[test]
553 fn test_query_error_from_parser_error() {
554 let parser_error = ParserError::ParserError("syntax error".to_string());
555 let query_error = QueryError::Parser { source: parser_error };
556
557 assert!(query_error.to_string().contains("syntax error"));
558 }
559
560 #[test]
561 fn test_resolved_table() {
562 let table = ResolvedTable {
563 table: "my_table".to_string(),
564 };
565
566 assert_eq!(table.table(), "my_table");
567 assert_eq!(table.to_string(), "my_table");
568 }
569
570 #[test]
571 fn test_resolved_table_clone_and_eq() {
572 let table1 = ResolvedTable {
573 table: "table1".to_string(),
574 };
575 let table2 = table1.clone();
576 let table3 = ResolvedTable {
577 table: "table2".to_string(),
578 };
579
580 assert_eq!(table1, table2);
581 assert_ne!(table1, table3);
582 }
583}