1use crate::{
16 PrepareSelectObjectSnapshotError, QueryError, SELECT_DEFAULT_READ_BUFFER_SIZE, SelectError, SelectGetObjectReader,
17 SelectInputMetrics, SelectObjectOptions, SelectObjectSnapshot, SelectObjectSnapshotReadError, SelectStorageError,
18 SelectStore, SnapshotConsistencyError,
19 input_stream::{CompressionFormat, compressed_input_reader, compressed_input_stream, processed_bytes_limit},
20 query::{
21 ast::{JsonPathSegment, JsonSource},
22 parser::RustFsDialect,
23 session::QueryExecutionTracker,
24 },
25 resolve_select_object_store_handle, select_is_err_bucket_not_found, select_is_err_object_not_found,
26 select_is_err_version_not_found,
27};
28use async_trait::async_trait;
29use bytes::Bytes;
30use chrono::{DateTime, Utc};
31use datafusion::{
32 execution::memory_pool::{MemoryPool, UnboundedMemoryPool},
33 object_store::{
34 Attributes, CopyOptions, Error as o_Error, GetOptions, GetRange, GetResult, GetResultPayload, ListResult,
35 MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, path::Path,
36 },
37 sql::{
38 planner::IdentNormalizer,
39 sqlparser::{
40 ast::{Expr, Ident, JsonPathElem, ObjectNamePart, SetExpr, Statement, TableFactor},
41 parser::Parser as SqlParser,
42 },
43 },
44};
45use futures::pin_mut;
46use futures::{Stream, StreamExt, TryStreamExt, future::ready, stream};
47use futures_core::stream::BoxStream;
48use http::{HeaderMap, HeaderValue, header::HeaderName};
49use memchr::{memchr, memmem};
50use rustfs_common::DEFAULT_DELIMITER;
51use s3s::header::{
52 X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
53 X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
54};
55use s3s::{
56 S3Error, S3ErrorCode, S3Result,
57 dto::{CompressionType, InputSerialization, ScanRange, SelectObjectContentInput},
58};
59use std::collections::VecDeque;
60use std::ops::Range;
61use std::sync::Arc;
62#[cfg(test)]
63use std::sync::atomic::{AtomicUsize, Ordering};
64use tokio::{io::AsyncReadExt, sync::OnceCell};
65use tokio_util::io::ReaderStream;
66use transform_stream::AsyncTryStream;
67
68use crate::csv_input::{CsvSyntax, csv_input_requires_normalization, normalize_csv_stream};
69use crate::storage_api::object_store::HTTPRangeSpec;
70
71mod json_document;
72
73use json_document::{compressed_json_document_ndjson_stream, json_document_ndjson_stream};
74
75fn select_default_read_buffer_size_u64() -> u64 {
76 u64::try_from(SELECT_DEFAULT_READ_BUFFER_SIZE).unwrap_or(u64::MAX)
77}
78
79fn compression_format(input: &InputSerialization) -> Result<Option<CompressionFormat>, SelectError> {
80 match input.compression_type.as_ref().map(|value| value.as_str()) {
81 None | Some(CompressionType::NONE) => Ok(None),
82 Some(CompressionType::GZIP) => Ok(Some(CompressionFormat::Gzip)),
83 Some(CompressionType::BZIP2) => Ok(Some(CompressionFormat::Bzip2)),
84 Some(_) => Err(SelectError::InvalidCompressionFormat),
85 }
86}
87
88pub const MAX_JSON_DOCUMENT_BYTES: u64 = 128 * 1024 * 1024;
94#[cfg(test)]
95use json_document::{
96 JSON_CANCELLATION_CHECK_BYTES, JSON_CANCELLATION_CHECK_KEYS, escaped_json_object_with_size, flatten_json_document_to_ndjson,
97 json_key_eq_ignore_ascii_case_with_checkpoint, parse_json_document_to_lines, remove_json_source_key_with_checkpoint,
98 validate_json_document_size,
99};
100pub const INVALID_SCAN_RANGE_MESSAGE: &str =
101 "The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again.";
102const NORMALIZED_RECORD_DELIMITER: &[u8] = b"\r\n";
103const NORMALIZED_FIELD_DELIMITER: &[u8] = &[DEFAULT_DELIMITER];
104
105pub fn is_noop_scan_range(scan_range: &ScanRange) -> bool {
107 scan_range.start == Some(0) && scan_range.end.is_none()
108}
109
110pub struct EcObjectStore {
111 input: Arc<SelectObjectContentInput>,
112 need_convert: bool,
113 delimiter: String,
114 is_json_document: bool,
118 json_source: JsonSource,
120 input_metrics: Arc<SelectInputMetrics>,
121 memory_pool: Arc<dyn MemoryPool>,
122 query_tracker: Option<QueryExecutionTracker>,
123 store: Option<Arc<SelectStore>>,
124 snapshot: OnceCell<Arc<SelectObjectSnapshot>>,
125 #[cfg(test)]
126 reader_open_count: Arc<AtomicUsize>,
127}
128
129#[derive(Debug, thiserror::Error)]
130pub(crate) enum EcObjectStoreBuildError {
131 #[error("ec store not inited")]
132 StoreUnavailable,
133 #[error("SelectObjectContent snapshot consistency failure: {0}")]
134 Snapshot(#[source] SnapshotConsistencyError),
135}
136
137#[derive(Debug, thiserror::Error)]
138pub(crate) enum SelectObjectStoreError {
139 #[error("SelectObjectContent bucket does not exist")]
140 BucketNotFound {
141 #[source]
142 source: SelectStorageError,
143 },
144 #[error("SelectObjectContent object does not exist")]
145 ObjectNotFound {
146 #[source]
147 source: SelectStorageError,
148 },
149 #[error("SelectObjectContent storage failure")]
150 Storage {
151 #[source]
152 source: SelectStorageError,
153 },
154 #[error("SelectObjectContent ScanRange is invalid")]
155 InvalidScanRange,
156}
157
158impl SelectObjectStoreError {
159 pub(crate) fn select_error(&self) -> SelectError {
160 match self {
161 Self::BucketNotFound { .. } => SelectError::BucketNotFound,
162 Self::ObjectNotFound { .. } => SelectError::ObjectNotFound,
163 Self::InvalidScanRange => SelectError::InvalidScanRange,
164 Self::Storage { .. } => SelectError::InternalError,
165 }
166 }
167}
168
169#[derive(Clone, Copy, Debug)]
170pub struct SelectScanRange {
171 start: u64,
172 end: u64,
173}
174
175impl SelectScanRange {
176 pub const fn new(start: u64, end: u64) -> Self {
177 Self { start, end }
178 }
179
180 pub const fn start(&self) -> u64 {
181 self.start
182 }
183
184 pub const fn end(&self) -> u64 {
185 self.end
186 }
187}
188
189#[derive(Clone, Copy, Debug)]
190pub struct InvalidScanRange;
191
192impl EcObjectStore {
193 pub fn new(input: Arc<SelectObjectContentInput>) -> S3Result<Self> {
194 let source = legacy_json_source_from_input(&input);
195 Self::build_lazy(
196 input,
197 Arc::new(UnboundedMemoryPool::default()),
198 None,
199 Arc::new(SelectInputMetrics::default()),
200 source,
201 )
202 .map_err(map_build_error_to_s3)
203 }
204
205 pub fn new_with_snapshot(input: Arc<SelectObjectContentInput>, snapshot: Arc<SelectObjectSnapshot>) -> S3Result<Self> {
206 let source = legacy_json_source_from_input(&input);
207 Self::build_with_snapshot(
208 input,
209 Arc::new(UnboundedMemoryPool::default()),
210 None,
211 Arc::new(SelectInputMetrics::default()),
212 snapshot,
213 source,
214 )
215 .map_err(map_build_error_to_s3)
216 }
217
218 pub(crate) fn new_with_memory_pool_and_source(
219 input: Arc<SelectObjectContentInput>,
220 memory_pool: Arc<dyn MemoryPool>,
221 input_metrics: Arc<SelectInputMetrics>,
222 snapshot: Option<Arc<SelectObjectSnapshot>>,
223 source: JsonSource,
224 ) -> std::result::Result<Self, EcObjectStoreBuildError> {
225 match snapshot {
226 Some(snapshot) => Self::build_with_snapshot(input, memory_pool, None, input_metrics, snapshot, source),
227 None => Self::build_lazy(input, memory_pool, None, input_metrics, source),
228 }
229 }
230
231 pub(crate) fn new_with_query_tracker_and_source(
232 input: Arc<SelectObjectContentInput>,
233 memory_pool: Arc<dyn MemoryPool>,
234 query_tracker: QueryExecutionTracker,
235 input_metrics: Arc<SelectInputMetrics>,
236 snapshot: Option<Arc<SelectObjectSnapshot>>,
237 source: JsonSource,
238 ) -> std::result::Result<Self, EcObjectStoreBuildError> {
239 match snapshot {
240 Some(snapshot) => Self::build_with_snapshot(input, memory_pool, Some(query_tracker), input_metrics, snapshot, source),
241 None => Self::build_lazy(input, memory_pool, Some(query_tracker), input_metrics, source),
242 }
243 }
244
245 fn build_lazy(
246 input: Arc<SelectObjectContentInput>,
247 memory_pool: Arc<dyn MemoryPool>,
248 query_tracker: Option<QueryExecutionTracker>,
249 input_metrics: Arc<SelectInputMetrics>,
250 source: JsonSource,
251 ) -> std::result::Result<Self, EcObjectStoreBuildError> {
252 let store = resolve_select_object_store_handle().ok_or(EcObjectStoreBuildError::StoreUnavailable)?;
253 Ok(Self::build(input, memory_pool, query_tracker, input_metrics, Some(store), None, source))
254 }
255
256 fn build_with_snapshot(
257 input: Arc<SelectObjectContentInput>,
258 memory_pool: Arc<dyn MemoryPool>,
259 query_tracker: Option<QueryExecutionTracker>,
260 input_metrics: Arc<SelectInputMetrics>,
261 snapshot: Arc<SelectObjectSnapshot>,
262 source: JsonSource,
263 ) -> std::result::Result<Self, EcObjectStoreBuildError> {
264 if !snapshot.is_for(&input.bucket, &input.key) {
265 return Err(EcObjectStoreBuildError::Snapshot(SnapshotConsistencyError::ObjectChanged));
266 }
267 Ok(Self::build(
268 input,
269 memory_pool,
270 query_tracker,
271 input_metrics,
272 None,
273 Some(snapshot),
274 source,
275 ))
276 }
277
278 fn build(
279 input: Arc<SelectObjectContentInput>,
280 memory_pool: Arc<dyn MemoryPool>,
281 query_tracker: Option<QueryExecutionTracker>,
282 input_metrics: Arc<SelectInputMetrics>,
283 store: Option<Arc<SelectStore>>,
284 snapshot: Option<Arc<SelectObjectSnapshot>>,
285 source: JsonSource,
286 ) -> Self {
287 let (need_convert, delimiter) = if let Some(csv) = input.request.input_serialization.csv.as_ref() {
288 if let Some(delimiter) = csv.field_delimiter.as_ref() {
289 if delimiter.len() > 1 {
290 (true, delimiter.to_owned())
291 } else {
292 (false, String::new())
293 }
294 } else {
295 (false, String::new())
296 }
297 } else {
298 (false, String::new())
299 };
300
301 let is_json_document = is_json_document_input(&input);
304
305 Self {
306 input,
307 need_convert,
308 delimiter,
309 is_json_document,
310 json_source: source,
311 input_metrics,
312 memory_pool,
313 query_tracker,
314 store,
315 snapshot: match snapshot {
316 Some(snapshot) => OnceCell::new_with(Some(snapshot)),
317 None => OnceCell::new(),
318 },
319 #[cfg(test)]
320 reader_open_count: Arc::new(AtomicUsize::new(0)),
321 }
322 }
323
324 fn scan_range(&self, object_size: u64) -> Result<Option<SelectScanRange>> {
325 let Some(scan_range) = self.input.request.scan_range.as_ref() else {
326 return Ok(None);
327 };
328 if is_noop_scan_range(scan_range) {
329 return Ok(None);
330 }
331 scan_range_from_bounds(scan_range.start, scan_range.end, object_size)
332 }
333
334 fn record_delimiter(&self) -> Vec<u8> {
335 self.input
336 .request
337 .input_serialization
338 .csv
339 .as_ref()
340 .and_then(|csv| csv.record_delimiter.as_ref())
341 .map(|delimiter| delimiter.as_bytes().to_vec())
342 .unwrap_or_else(|| b"\n".to_vec())
343 }
344
345 fn record_delimiter_for_conversion(&self) -> Option<Vec<u8>> {
346 let delimiter = self.record_delimiter();
347 (self.need_convert || (delimiter.len() == 2 && delimiter != NORMALIZED_RECORD_DELIMITER)).then_some(delimiter)
348 }
349
350 fn convert_csv_stream<S>(&self, stream: S) -> BoxStream<'static, Result<Bytes>>
351 where
352 S: Stream<Item = Result<Bytes>> + Send + 'static,
353 {
354 if let Some(csv) = self.input.request.input_serialization.csv.as_ref()
355 && csv_input_requires_normalization(csv.quote_character.as_deref(), csv.quote_escape_character.as_deref())
356 {
357 let syntax = CsvSyntax {
358 quote: csv.quote_character.as_deref(),
359 escape: csv.quote_escape_character.as_deref(),
360 field: csv.field_delimiter.as_deref(),
361 record: csv.record_delimiter.as_deref(),
362 comment: csv.comments.as_ref().and_then(|comment| comment.as_bytes().first().copied()),
363 };
364 return normalize_csv_stream(stream, &syntax);
365 }
366 convert_csv_delimiter_stream(
367 stream,
368 self.record_delimiter_for_conversion(),
369 self.need_convert.then(|| self.delimiter.clone()),
370 )
371 }
372
373 fn csv_has_header(&self) -> bool {
374 self.input
375 .request
376 .input_serialization
377 .csv
378 .as_ref()
379 .and_then(|csv| csv.file_header_info.as_ref())
380 .is_some_and(|info| matches!(info.as_str(), "USE" | "IGNORE"))
381 }
382
383 async fn snapshot(&self, version: Option<&str>) -> Result<&Arc<SelectObjectSnapshot>> {
384 let snapshot = self
385 .snapshot
386 .get_or_try_init(|| async {
387 let store = self.store.as_ref().ok_or_else(|| o_Error::Generic {
388 store: "EcObjectStore",
389 source: "prepared snapshot is unavailable".into(),
390 })?;
391 let opts = SelectObjectOptions {
392 version_id: version.map(|version| {
393 let version = version.trim();
394 if version.eq_ignore_ascii_case("null") {
395 uuid::Uuid::nil().to_string()
396 } else {
397 version.to_owned()
398 }
399 }),
400 ..Default::default()
401 };
402 let snapshot = store
403 .prepare_select_object_snapshot(&self.input.bucket, &self.input.key, &select_read_headers(&self.input), &opts)
404 .await
405 .map_err(|err| map_prepare_snapshot_error(&self.input.bucket, &self.input.key, err))?;
406 Ok::<_, o_Error>(Arc::new(snapshot))
407 })
408 .await?;
409 if let Some(requested) = version
410 && !snapshot.matches_version(requested)
411 {
412 return Err(o_Error::Generic {
413 store: "EcObjectStore",
414 source: "prepared snapshot is pinned to a different object version".into(),
415 });
416 }
417 Ok(snapshot)
418 }
419
420 async fn object_reader(&self, range: Option<HTTPRangeSpec>) -> Result<SelectGetObjectReader> {
421 #[cfg(test)]
422 self.reader_open_count.fetch_add(1, Ordering::Relaxed);
423 self.snapshot(None)
424 .await?
425 .open_reader(range)
426 .await
427 .map_err(|err| snapshot_read_error(&self.input.bucket, &self.input.key, err))
428 }
429
430 async fn read_raw_range(&self, range: Range<u64>) -> Result<Bytes> {
431 if range.is_empty() {
432 return Ok(Bytes::new());
433 }
434 let snapshot = self.snapshot(None).await?;
435 let reader = self.object_reader(Some(http_range_spec_from_range(range.clone()))).await?;
436 let object_size = snapshot.logical_size();
437 let resolved_range = GetRange::Bounded(range)
438 .as_range(object_size)
439 .map_err(|err| o_Error::Generic {
440 store: "EcObjectStore",
441 source: Box::new(err),
442 })?;
443 let expected_size = usize::try_from(resolved_range.end - resolved_range.start).map_err(|err| o_Error::Generic {
444 store: "EcObjectStore",
445 source: Box::new(err),
446 })?;
447 let mut reader = reader.stream;
448 let mut bytes = Vec::new();
449 reader.read_to_end(&mut bytes).await.map_err(|err| o_Error::Generic {
450 store: "EcObjectStore",
451 source: Box::new(err),
452 })?;
453 if bytes.len() < expected_size {
454 return Err(incomplete_object_stream_error(expected_size - bytes.len()));
455 }
456 bytes.truncate(expected_size);
457 Ok(Bytes::from(bytes))
458 }
459
460 async fn read_header_record(&self, object_size: u64, delimiter: &[u8]) -> Result<Bytes> {
461 if object_size == 0 {
462 return Ok(Bytes::new());
463 }
464
465 let mut end = select_default_read_buffer_size_u64().min(object_size);
466 loop {
467 let bytes = self.read_raw_range(0..end).await?;
468 if let Some(pos) = find_delimiter(&bytes, delimiter) {
469 return Ok(bytes.slice(0..pos + delimiter.len()));
470 }
471 if end == object_size {
472 return Ok(bytes);
473 }
474 end = end.saturating_mul(2).min(object_size);
475 }
476 }
477
478 async fn scan_range_read_start(&self, scan_range: SelectScanRange, delimiter: &[u8]) -> Result<u64> {
479 let delimiter_len = u64::try_from(delimiter.len()).unwrap_or(u64::MAX);
480 let fallback_start = scan_range.start().saturating_sub(delimiter_len);
481 if delimiter.len() != 2 || delimiter[0] != delimiter[1] || scan_range.start() == 0 {
482 return Ok(fallback_start);
483 }
484
485 let context_start = scan_range.start().saturating_sub(select_default_read_buffer_size_u64());
486 let context = self.read_raw_range(context_start..scan_range.start()).await?;
487 let suffix_len = context.iter().rev().take_while(|byte| **byte == delimiter[0]).count();
488 if suffix_len == context.len() && context_start > 0 {
489 return Err(o_Error::Generic {
490 store: "EcObjectStore",
491 source: "self-overlapping CSV record delimiter exceeds the bounded ScanRange context".into(),
492 });
493 }
494 if suffix_len == 0 {
495 return Ok(fallback_start);
496 }
497 Ok(scan_range.start().saturating_sub(u64::from(suffix_len % 2 != 0)))
498 }
499}
500
501pub(crate) fn is_json_document_input(input: &SelectObjectContentInput) -> bool {
502 input
503 .request
504 .input_serialization
505 .json
506 .as_ref()
507 .and_then(|json| json.type_.as_ref())
508 .is_some_and(|json_type| json_type.as_str() == "DOCUMENT")
509}
510
511pub(crate) fn legacy_json_source_from_input(input: &SelectObjectContentInput) -> JsonSource {
513 if !is_json_document_input(input) {
514 return JsonSource::default();
515 }
516 let Ok(mut statements) = SqlParser::parse_sql(&RustFsDialect, &input.request.expression) else {
517 return JsonSource::default();
518 };
519 if statements.len() != 1 {
520 return JsonSource::default();
521 }
522 let Some(Statement::Query(query)) = statements.pop() else {
523 return JsonSource::default();
524 };
525 let SetExpr::Select(select) = query.body.as_ref() else {
526 return JsonSource::default();
527 };
528 let [table] = select.from.as_slice() else {
529 return JsonSource::default();
530 };
531 let TableFactor::Table {
532 name, alias, json_path, ..
533 } = &table.relation
534 else {
535 return JsonSource::default();
536 };
537 let Some(ObjectNamePart::Identifier(table_name)) = name.0.first() else {
538 return JsonSource::default();
539 };
540 if name.0.len() > 2 {
541 return JsonSource::default();
542 }
543 let is_s3_object = if table_name.quote_style.is_some() {
544 table_name.value == "S3Object"
545 } else {
546 table_name.value.eq_ignore_ascii_case("S3Object")
547 };
548 if !is_s3_object {
549 return JsonSource::default();
550 }
551 let path = match (name.0.get(1), json_path.as_ref()) {
552 (Some(ObjectNamePart::Identifier(sub_path)), None) => vec![JsonPathSegment::Key {
553 name: sub_path.value.clone(),
554 quoted: sub_path.quote_style.is_some(),
555 }],
556 (None, None) => Vec::new(),
557 (None, Some(json_path)) if matches!(json_path.path.as_slice(), [JsonPathElem::Bracket { key: Expr::Wildcard(_) }]) => {
558 vec![JsonPathSegment::ArrayWildcard]
559 }
560 _ => return JsonSource::default(),
561 };
562 let scalar_column = alias
563 .as_ref()
564 .map(|alias| IdentNormalizer::default().normalize(alias.name.clone()))
565 .or_else(|| match path.as_slice() {
566 [] => Some(IdentNormalizer::default().normalize(table_name.clone())),
567 [JsonPathSegment::Key { name, quoted }] => {
568 let alias = if *quoted {
569 Ident::with_quote('"', name)
570 } else {
571 Ident::new(name)
572 };
573 Some(IdentNormalizer::default().normalize(alias))
574 }
575 [JsonPathSegment::ArrayWildcard] => Some(IdentNormalizer::default().normalize(Ident::new("_1"))),
576 _ => None,
577 });
578 JsonSource::new(path, scalar_column)
579}
580
581impl std::fmt::Debug for EcObjectStore {
582 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
583 f.debug_struct("EcObjectStore")
584 .field("bucket", &self.input.bucket)
585 .field("object", &self.input.key)
586 .field("need_convert", &self.need_convert)
587 .field("is_json_document", &self.is_json_document)
588 .field("json_source", &self.json_source)
589 .finish_non_exhaustive()
590 }
591}
592
593impl std::fmt::Display for EcObjectStore {
594 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
595 f.write_str("EcObjectStore")
596 }
597}
598
599fn unsupported_store_error(op: &str) -> o_Error {
600 o_Error::Generic {
601 store: "s3select-api",
602 source: Box::new(std::io::Error::new(
603 std::io::ErrorKind::Unsupported,
604 format!("operation {op} is not supported in EcObjectStore"),
605 )),
606 }
607}
608
609fn insert_header(headers: &mut HeaderMap, name: HeaderName, value: Option<&str>) {
610 if let Some(value) = value
611 && let Ok(value) = HeaderValue::from_str(value)
612 {
613 headers.insert(name, value);
614 }
615}
616
617fn select_read_headers(input: &SelectObjectContentInput) -> HeaderMap {
618 let mut headers = HeaderMap::new();
619 insert_header(
620 &mut headers,
621 X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
622 input.sse_customer_algorithm.as_deref(),
623 );
624 insert_header(&mut headers, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, input.sse_customer_key.as_deref());
625 insert_header(
626 &mut headers,
627 X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
628 input.sse_customer_key_md5.as_deref(),
629 );
630 headers
631}
632
633fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
634 match range {
635 GetRange::Bounded(range) => http_range_spec_from_range(range.clone()),
636 GetRange::Offset(start) => HTTPRangeSpec {
637 is_suffix_length: false,
638 start: *start as i64,
639 end: -1,
640 },
641 GetRange::Suffix(length) => HTTPRangeSpec {
642 is_suffix_length: true,
643 start: *length as i64,
644 end: -1,
645 },
646 }
647}
648
649fn http_range_spec_from_range(range: Range<u64>) -> HTTPRangeSpec {
650 HTTPRangeSpec {
651 is_suffix_length: false,
652 start: range.start as i64,
653 end: range.end.saturating_sub(1) as i64,
654 }
655}
656
657fn http_range_spec_from_start(start: u64) -> HTTPRangeSpec {
658 HTTPRangeSpec {
659 is_suffix_length: false,
660 start: start as i64,
661 end: -1,
662 }
663}
664
665enum DelimiterSearcher {
666 Empty,
667 Byte(u8),
668 Multi(Box<memmem::Finder<'static>>),
669}
670
671impl DelimiterSearcher {
672 fn new(delimiter: &[u8]) -> Self {
673 match delimiter {
674 [] => Self::Empty,
675 [byte] => Self::Byte(*byte),
676 _ => Self::Multi(Box::new(memmem::Finder::new(delimiter).into_owned())),
677 }
678 }
679
680 fn len(&self) -> usize {
681 match self {
682 Self::Empty => 0,
683 Self::Byte(_) => 1,
684 Self::Multi(finder) => finder.needle().len(),
685 }
686 }
687
688 fn find(&self, bytes: &[u8]) -> Option<usize> {
689 match self {
690 Self::Empty => None,
691 Self::Byte(byte) => memchr(*byte, bytes),
692 Self::Multi(finder) => finder.find(bytes),
693 }
694 }
695}
696
697fn find_delimiter(bytes: &[u8], delimiter: &[u8]) -> Option<usize> {
698 match delimiter {
699 [] => None,
700 [byte] => memchr(*byte, bytes),
701 _ => memmem::find(bytes, delimiter),
702 }
703}
704
705fn map_prepare_snapshot_error(bucket: &str, object: &str, err: PrepareSelectObjectSnapshotError) -> o_Error {
706 match err {
707 PrepareSelectObjectSnapshotError::Storage(err) => map_storage_error(bucket, object, err),
708 err => o_Error::Generic {
709 store: "EcObjectStore",
710 source: Box::new(err),
711 },
712 }
713}
714
715fn map_build_error_to_s3(error: EcObjectStoreBuildError) -> S3Error {
716 let mut s3_error = S3Error::with_message(S3ErrorCode::InternalError, SelectError::InternalError.to_string());
717 s3_error.set_source(Box::new(error));
718 s3_error
719}
720
721fn snapshot_read_error(bucket: &str, object: &str, err: SelectObjectSnapshotReadError) -> o_Error {
722 match err {
723 SelectObjectSnapshotReadError::Storage(err) => map_storage_error(bucket, object, err),
724 err => o_Error::Generic {
725 store: "EcObjectStore",
726 source: Box::new(err),
727 },
728 }
729}
730
731fn map_storage_error(bucket: &str, object: &str, err: SelectStorageError) -> o_Error {
732 if select_is_err_bucket_not_found(&err) {
733 return o_Error::NotFound {
734 path: format!("{bucket}/{object}"),
735 source: Box::new(SelectObjectStoreError::BucketNotFound { source: err }),
736 };
737 }
738 if select_is_err_object_not_found(&err) || select_is_err_version_not_found(&err) {
739 return o_Error::NotFound {
740 path: format!("{bucket}/{object}"),
741 source: Box::new(SelectObjectStoreError::ObjectNotFound { source: err }),
742 };
743 }
744 o_Error::Generic {
745 store: "EcObjectStore",
746 source: Box::new(SelectObjectStoreError::Storage { source: err }),
747 }
748}
749
750fn snapshot_last_modified(snapshot: &SelectObjectSnapshot) -> Result<DateTime<Utc>> {
751 let mod_time = snapshot.object_info().mod_time.ok_or_else(|| o_Error::Generic {
752 store: "EcObjectStore",
753 source: std::io::Error::new(std::io::ErrorKind::InvalidData, "snapshot metadata has no modification time").into(),
754 })?;
755 DateTime::<Utc>::from_timestamp(mod_time.unix_timestamp(), mod_time.nanosecond()).ok_or_else(|| o_Error::Generic {
756 store: "EcObjectStore",
757 source: std::io::Error::new(std::io::ErrorKind::InvalidData, "snapshot modification time is out of range").into(),
758 })
759}
760
761pub fn scan_range_from_bounds(start: Option<i64>, end: Option<i64>, object_size: u64) -> Result<Option<SelectScanRange>> {
762 parse_scan_range_from_bounds(start, end, object_size).map_err(|_| invalid_scan_range_store_error())
763}
764
765pub fn validate_scan_range_bounds(
766 start: Option<i64>,
767 end: Option<i64>,
768 object_size: u64,
769) -> std::result::Result<(), InvalidScanRange> {
770 parse_scan_range_from_bounds(start, end, object_size).map(|_| ())
771}
772
773fn parse_scan_range_from_bounds(
774 start: Option<i64>,
775 end: Option<i64>,
776 object_size: u64,
777) -> std::result::Result<Option<SelectScanRange>, InvalidScanRange> {
778 if start.is_none() && end.is_none() {
779 return Ok(None);
780 }
781 if start.is_some_and(|value| value < 0) || end.is_some_and(|value| value < 0) {
782 return Err(InvalidScanRange);
783 }
784 if let (Some(start), Some(end)) = (start, end)
785 && start > end
786 {
787 return Err(InvalidScanRange);
788 }
789 if let Some(start) = start {
790 let start = start as u64;
791 if object_size == 0 {
792 if start > 0 {
793 return Err(InvalidScanRange);
794 }
795 return Ok(Some(SelectScanRange::new(0, 0)));
796 }
797 if start >= object_size {
798 return Err(InvalidScanRange);
799 }
800 }
801 if object_size == 0 {
802 return Ok(Some(SelectScanRange::new(0, 0)));
803 }
804
805 let last_byte = object_size - 1;
806 let (start, end) = match (start, end) {
807 (Some(start), Some(end)) => (start as u64, (end as u64).min(last_byte)),
808 (Some(start), None) => (start as u64, last_byte),
809 (None, Some(suffix_len)) => {
810 let suffix_len = suffix_len as u64;
811 (object_size.saturating_sub(suffix_len), last_byte)
812 }
813 (None, None) => return Ok(None),
814 };
815 Ok(Some(SelectScanRange::new(start, end)))
816}
817
818fn invalid_scan_range_store_error() -> o_Error {
819 o_Error::Generic {
820 store: "EcObjectStore",
821 source: Box::new(SelectObjectStoreError::InvalidScanRange),
822 }
823}
824
825#[async_trait]
826impl ObjectStore for EcObjectStore {
827 async fn put_opts(&self, _location: &Path, _payload: PutPayload, _opts: PutOptions) -> Result<PutResult> {
828 Err(unsupported_store_error("put_opts"))
829 }
830
831 async fn put_multipart_opts(&self, _location: &Path, _opts: PutMultipartOptions) -> Result<Box<dyn MultipartUpload>> {
832 Err(unsupported_store_error("put_multipart_opts"))
833 }
834
835 async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
836 let snapshot = self.snapshot(options.version.as_deref()).await?;
840 let original_size = snapshot.logical_size();
841 let compression = compression_format(&self.input.request.input_serialization).map_err(|source| o_Error::Generic {
842 store: "EcObjectStore",
843 source: Box::new(source),
844 })?;
845 let has_effective_request_range = self
846 .input
847 .request
848 .scan_range
849 .as_ref()
850 .is_some_and(|scan_range| !is_noop_scan_range(scan_range));
851 if compression.is_some() && (options.range.is_some() || has_effective_request_range) {
852 return Err(o_Error::Generic {
853 store: "EcObjectStore",
854 source: Box::new(SelectError::UnsupportedScanRangeInput),
855 });
856 }
857 let object_info = snapshot.object_info();
858 let meta = ObjectMeta {
859 location: location.clone(),
860 last_modified: snapshot_last_modified(snapshot)?,
861 size: original_size,
862 e_tag: object_info.etag.clone(),
863 version: object_info.version_id.map(|version| version.to_string()),
864 };
865 let result_range = match options.range.as_ref() {
866 Some(range) => range.as_range(original_size).map_err(|err| o_Error::Generic {
867 store: "EcObjectStore",
868 source: Box::new(err),
869 })?,
870 None => 0..original_size,
871 };
872 if options.head {
873 return Ok(GetResult {
874 payload: GetResultPayload::Stream(stream::empty().boxed()),
875 meta,
876 range: result_range,
877 attributes: Attributes::default(),
878 });
879 }
880
881 let needs_scan_context = options.range.is_none() && has_effective_request_range;
882 let scan_context = if needs_scan_context {
883 if let Some(scan_range) = self.scan_range(original_size)? {
884 let delimiter = self.record_delimiter();
885 let read_start = self.scan_range_read_start(scan_range, &delimiter).await?;
886 Some((scan_range, read_start))
887 } else {
888 None
889 }
890 } else {
891 None
892 };
893
894 let range = options.range.as_ref().map(http_range_spec_from_get_range);
895 let reader = if let Some((_, read_start)) = scan_context.as_ref() {
896 let range = (original_size > 0).then(|| http_range_spec_from_start(*read_start));
897 self.object_reader(range).await?
898 } else {
899 self.object_reader(range).await?
900 };
901
902 let meter_input = self.input.request.input_serialization.parquet.is_none();
903 let payload = if let Some(compression) = compression {
904 let max_processed_bytes = processed_bytes_limit();
905 let query_guard = match self.query_tracker.as_ref() {
906 Some(query_tracker) => Some(query_tracker.query_guard().ok_or_else(|| o_Error::Generic {
907 store: "EcObjectStore",
908 source: Box::new(QueryError::Cancel),
909 })?),
910 None => None,
911 };
912 if self.is_json_document {
913 let reader = compressed_input_reader(
914 reader.stream,
915 original_size,
916 compression,
917 Arc::clone(&self.input_metrics),
918 max_processed_bytes,
919 query_guard,
920 );
921 let stream = compressed_json_document_ndjson_stream(
922 reader,
923 self.json_source.clone(),
924 Arc::clone(&self.memory_pool),
925 self.query_tracker.clone(),
926 );
927 GetResultPayload::Stream(stream)
928 } else {
929 let input_record_delimiter = if self.input.request.input_serialization.csv.is_some() {
930 self.record_delimiter()
931 } else {
932 b"\n".to_vec()
933 };
934 let stream = compressed_input_stream(
935 reader.stream,
936 original_size,
937 compression,
938 Arc::clone(&self.input_metrics),
939 input_record_delimiter,
940 max_processed_bytes,
941 query_guard,
942 )?;
943 let stream = self.convert_csv_stream(stream);
944 GetResultPayload::Stream(stream)
945 }
946 } else if options.range.is_some() {
947 let size = usize::try_from(result_range.end - result_range.start).map_err(|err| o_Error::Generic {
948 store: "EcObjectStore",
949 source: Box::new(err),
950 })?;
951 let stream = bytes_stream(ReaderStream::with_capacity(reader.stream, SELECT_DEFAULT_READ_BUFFER_SIZE), size);
952 if meter_input {
953 GetResultPayload::Stream(meter_uncompressed_input_stream(stream, Arc::clone(&self.input_metrics)).boxed())
954 } else {
955 GetResultPayload::Stream(stream.boxed())
956 }
957 } else if self.is_json_document {
958 let stream = json_document_ndjson_stream(
960 reader.stream,
961 original_size,
962 self.json_source.clone(),
963 Arc::clone(&self.input_metrics),
964 Arc::clone(&self.memory_pool),
965 self.query_tracker.clone(),
966 )?;
967 GetResultPayload::Stream(stream)
968 } else if let Some((scan_range, read_start)) = scan_context {
969 let delimiter = self.record_delimiter();
970 let include_header = self.csv_has_header();
971 let header = if include_header && read_start > 0 {
972 let input_metrics = self.input_metrics.recorder();
973 let header = self.read_header_record(original_size, &delimiter).await?;
974 input_metrics.record_uncompressed(header.len());
975 Some(header)
976 } else {
977 None
978 };
979 let stream = scan_range_stream(
980 meter_uncompressed_input_stream(
981 ReaderStream::with_capacity(reader.stream, SELECT_DEFAULT_READ_BUFFER_SIZE),
982 Arc::clone(&self.input_metrics),
983 ),
984 delimiter,
985 scan_range,
986 include_header && header.is_none(),
987 read_start,
988 original_size,
989 )
990 .boxed();
991 let stream = if let Some(header) = header {
992 stream::once(ready(Ok(header))).chain(stream).boxed()
993 } else {
994 stream
995 };
996 let stream = self.convert_csv_stream(stream);
997 GetResultPayload::Stream(stream)
998 } else {
999 let stream_size = usize::try_from(original_size).map_err(|err| o_Error::Generic {
1000 store: "EcObjectStore",
1001 source: Box::new(err),
1002 })?;
1003 let stream = bytes_stream(ReaderStream::with_capacity(reader.stream, SELECT_DEFAULT_READ_BUFFER_SIZE), stream_size);
1004 if meter_input {
1005 let stream = meter_uncompressed_input_stream(stream, Arc::clone(&self.input_metrics));
1006 let stream = self.convert_csv_stream(stream);
1007 GetResultPayload::Stream(stream)
1008 } else {
1009 GetResultPayload::Stream(stream.boxed())
1010 }
1011 };
1012
1013 Ok(GetResult {
1014 payload,
1015 meta,
1016 range: result_range,
1017 attributes: Attributes::default(),
1018 })
1019 }
1020
1021 async fn get_ranges(&self, _location: &Path, ranges: &[Range<u64>]) -> Result<Vec<Bytes>> {
1022 let mut out = Vec::with_capacity(ranges.len());
1023 for range in ranges {
1024 out.push(self.read_raw_range(range.clone()).await?);
1025 }
1026 Ok(out)
1027 }
1028
1029 fn delete_stream(&self, _locations: BoxStream<'static, Result<Path>>) -> BoxStream<'static, Result<Path>> {
1030 stream::once(ready(Err(unsupported_store_error("delete_stream")))).boxed()
1031 }
1032
1033 fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
1034 stream::once(ready(Err(unsupported_store_error("list")))).boxed()
1035 }
1036
1037 async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> Result<ListResult> {
1038 Err(unsupported_store_error("list_with_delimiter"))
1039 }
1040
1041 async fn copy_opts(&self, _from: &Path, _to: &Path, _options: CopyOptions) -> Result<()> {
1042 Err(unsupported_store_error("copy_opts"))
1043 }
1044}
1045
1046struct CsvDelimiterConverter {
1047 record_delimiter: Option<Vec<u8>>,
1048 field_delimiter: Option<Vec<u8>>,
1049 carry: Vec<u8>,
1050}
1051
1052impl CsvDelimiterConverter {
1053 fn new(record_delimiter: Option<Vec<u8>>, field_delimiter: Option<Vec<u8>>) -> Self {
1054 Self {
1055 record_delimiter: record_delimiter.filter(|delimiter| !delimiter.is_empty()),
1056 field_delimiter: field_delimiter.filter(|delimiter| !delimiter.is_empty()),
1057 carry: Vec::new(),
1058 }
1059 }
1060
1061 fn max_delimiter_len(&self) -> usize {
1062 self.record_delimiter
1063 .as_ref()
1064 .into_iter()
1065 .chain(self.field_delimiter.as_ref())
1066 .map(Vec::len)
1067 .max()
1068 .unwrap_or(1)
1069 }
1070
1071 fn convert_prefix(&self, bytes: &[u8], end: usize) -> (Vec<u8>, usize) {
1072 let mut converted = Vec::with_capacity(bytes.len());
1073 let mut pos = 0;
1074 while pos < end {
1075 let record_match = self
1076 .record_delimiter
1077 .as_ref()
1078 .filter(|delimiter| bytes[pos..].starts_with(delimiter));
1079 let field_match = self
1080 .field_delimiter
1081 .as_ref()
1082 .filter(|delimiter| bytes[pos..].starts_with(delimiter));
1083 if let Some(delimiter) = field_match
1084 && record_match.is_none_or(|record_delimiter| delimiter.len() > record_delimiter.len())
1085 {
1086 converted.extend_from_slice(NORMALIZED_FIELD_DELIMITER);
1087 pos += delimiter.len();
1088 } else if let Some(delimiter) = record_match {
1089 if delimiter.len() == 2 && delimiter != NORMALIZED_RECORD_DELIMITER {
1090 converted.extend_from_slice(NORMALIZED_RECORD_DELIMITER);
1091 } else {
1092 converted.extend_from_slice(delimiter);
1093 }
1094 pos += delimiter.len();
1095 } else {
1096 converted.push(bytes[pos]);
1097 pos += 1;
1098 }
1099 }
1100 (converted, pos)
1101 }
1102
1103 fn convert_chunk(&mut self, chunk: &[u8]) -> Vec<u8> {
1104 let mut combined = Vec::with_capacity(self.carry.len() + chunk.len());
1105 combined.extend_from_slice(&self.carry);
1106 combined.extend_from_slice(chunk);
1107
1108 let safe_end = combined.len().saturating_sub(self.max_delimiter_len().saturating_sub(1));
1109 let (converted, pos) = self.convert_prefix(&combined, safe_end);
1110 self.carry.clear();
1111 self.carry.extend_from_slice(&combined[pos..]);
1112 converted
1113 }
1114
1115 fn finish(&mut self) -> Vec<u8> {
1116 let (converted, _) = self.convert_prefix(&self.carry, self.carry.len());
1117 self.carry.clear();
1118 converted
1119 }
1120}
1121
1122fn convert_delimiter_stream<S>(
1123 stream: S,
1124 record_delimiter: Option<Vec<u8>>,
1125 field_delimiter: Option<Vec<u8>>,
1126) -> BoxStream<'static, Result<Bytes>>
1127where
1128 S: Stream<Item = Result<Bytes>> + Send + 'static,
1129{
1130 AsyncTryStream::<Bytes, o_Error, _>::new(|mut y| async move {
1131 let mut converter = CsvDelimiterConverter::new(record_delimiter, field_delimiter);
1132 pin_mut!(stream);
1133 while let Some(result) = stream.next().await {
1134 let bytes = result?;
1135 let converted = converter.convert_chunk(&bytes);
1136 if !converted.is_empty() {
1137 y.yield_ok(Bytes::from(converted)).await;
1138 }
1139 }
1140 let converted = converter.finish();
1141 if !converted.is_empty() {
1142 y.yield_ok(Bytes::from(converted)).await;
1143 }
1144 Ok(())
1145 })
1146 .boxed()
1147}
1148
1149#[cfg(test)]
1150fn convert_record_delimiter_stream<S>(stream: S, delimiter: Vec<u8>) -> BoxStream<'static, Result<Bytes>>
1151where
1152 S: Stream<Item = Result<Bytes>> + Send + 'static,
1153{
1154 convert_delimiter_stream(stream, Some(delimiter), None)
1156}
1157
1158#[cfg(test)]
1159fn convert_field_delimiter_stream<S>(stream: S, delimiter: String) -> BoxStream<'static, Result<Bytes>>
1160where
1161 S: Stream<Item = Result<Bytes>> + Send + 'static,
1162{
1163 convert_delimiter_stream(stream, None, Some(delimiter.into_bytes()))
1164}
1165
1166fn convert_csv_delimiter_stream<S>(
1167 stream: S,
1168 record_delimiter: Option<Vec<u8>>,
1169 field_delimiter: Option<String>,
1170) -> BoxStream<'static, Result<Bytes>>
1171where
1172 S: Stream<Item = Result<Bytes>> + Send + 'static,
1173{
1174 match (record_delimiter, field_delimiter) {
1175 (None, None) => stream.boxed(),
1176 (record, field) => convert_delimiter_stream(stream, record, field.map(String::into_bytes)),
1177 }
1178}
1179
1180struct ScanRangeState<S> {
1181 stream: S,
1182 delimiter: DelimiterSearcher,
1183 range: SelectScanRange,
1184 include_header: bool,
1185 offset: u64,
1186 record_start: u64,
1187 record: Vec<u8>,
1188 pending: VecDeque<Bytes>,
1189 done: bool,
1190 expected_end: u64,
1191}
1192
1193fn scan_range_stream<S>(
1194 stream: S,
1195 delimiter: Vec<u8>,
1196 range: SelectScanRange,
1197 include_header: bool,
1198 base_offset: u64,
1199 expected_end: u64,
1200) -> BoxStream<'static, Result<Bytes>>
1201where
1202 S: Stream<Item = std::io::Result<Bytes>> + Send + Unpin + 'static,
1203{
1204 let state = ScanRangeState {
1205 stream,
1206 delimiter: DelimiterSearcher::new(&delimiter),
1207 range,
1208 include_header,
1209 offset: base_offset,
1210 record_start: base_offset,
1211 record: Vec::new(),
1212 pending: VecDeque::new(),
1213 done: false,
1214 expected_end,
1215 };
1216
1217 stream::unfold(state, |mut state| async move {
1218 loop {
1219 if let Some(bytes) = state.pending.pop_front() {
1220 return Some((Ok(bytes), state));
1221 }
1222 if state.done {
1223 return None;
1224 }
1225 match state.stream.next().await {
1226 Some(Ok(bytes)) => state.push_chunk(&bytes),
1227 Some(Err(err)) => {
1228 state.done = true;
1229 return Some((
1230 Err(o_Error::Generic {
1231 store: "EcObjectStore",
1232 source: Box::new(err),
1233 }),
1234 state,
1235 ));
1236 }
1237 None => {
1238 if state.offset < state.expected_end {
1239 state.done = true;
1240 return Some((Err(incomplete_object_stream_error(state.expected_end - state.offset)), state));
1241 }
1242 state.finish_pending_record();
1243 state.done = true;
1244 }
1245 }
1246 }
1247 })
1248 .boxed()
1249}
1250
1251impl<S> ScanRangeState<S> {
1252 fn push_chunk(&mut self, bytes: &[u8]) {
1253 if bytes.is_empty() || self.done {
1254 return;
1255 }
1256 if self.record.is_empty() {
1257 self.record_start = self.offset;
1258 }
1259 let delimiter_len = self.delimiter.len();
1260 let search_start = self.record.len().saturating_sub(delimiter_len.saturating_sub(1));
1261 self.record.extend_from_slice(bytes);
1262 self.offset = self.offset.saturating_add(bytes.len() as u64);
1263
1264 let mut record_begin = 0;
1265 let mut search_start = search_start;
1266 while let Some(pos) = self.delimiter.find(&self.record[search_start..]) {
1267 let record_end = search_start + pos + delimiter_len;
1268 let relative_start = u64::try_from(record_begin).unwrap_or(u64::MAX);
1269 let record_start = self.record_start.saturating_add(relative_start);
1270 self.finish_record(record_begin..record_end, record_start);
1271 record_begin = record_end;
1272 if self.done {
1273 break;
1274 }
1275 search_start = record_end;
1276 }
1277
1278 if record_begin > 0 {
1279 self.record.drain(..record_begin);
1280 let consumed = u64::try_from(record_begin).unwrap_or(u64::MAX);
1281 self.record_start = self.record_start.saturating_add(consumed);
1282 }
1283 }
1284
1285 fn finish_record(&mut self, record_range: Range<usize>, record_start: u64) {
1286 if self.should_emit_record(record_start) {
1287 self.pending.push_back(Bytes::copy_from_slice(&self.record[record_range]));
1288 }
1289 }
1290
1291 fn finish_pending_record(&mut self) {
1292 if self.record.is_empty() {
1293 return;
1294 }
1295 let record = std::mem::take(&mut self.record);
1296 let record_start = self.record_start;
1297 if self.should_emit_record(record_start) {
1298 self.pending.push_back(Bytes::from(record));
1299 }
1300 }
1301
1302 fn should_emit_record(&mut self, record_start: u64) -> bool {
1303 let include_header = self.include_header && record_start == 0;
1304 let include_record = record_start >= self.range.start() && record_start <= self.range.end();
1305 if include_header || include_record {
1306 return true;
1307 }
1308 if record_start > self.range.end() {
1309 self.done = true;
1310 }
1311 false
1312 }
1313}
1314
1315fn meter_uncompressed_input_stream<S, E>(
1316 stream: S,
1317 input_metrics: Arc<SelectInputMetrics>,
1318) -> impl Stream<Item = std::result::Result<Bytes, E>> + Send + 'static
1319where
1320 S: Stream<Item = std::result::Result<Bytes, E>> + Send + 'static,
1321 E: Send + 'static,
1322{
1323 let input_metrics = input_metrics.recorder();
1324 stream.inspect_ok(move |bytes| input_metrics.record_uncompressed(bytes.len()))
1325}
1326
1327pub fn bytes_stream<S>(stream: S, content_length: usize) -> impl Stream<Item = Result<Bytes>> + Send + 'static
1328where
1329 S: Stream<Item = Result<Bytes, std::io::Error>> + Send + 'static,
1330{
1331 AsyncTryStream::<Bytes, o_Error, _>::new(|mut y| async move {
1332 pin_mut!(stream);
1333 let mut remaining: usize = content_length;
1334 while remaining > 0 {
1335 let Some(result) = stream.next().await else {
1336 break;
1337 };
1338 let mut bytes = result.map_err(|e| o_Error::Generic {
1339 store: "",
1340 source: Box::new(e),
1341 })?;
1342 if bytes.len() > remaining {
1343 bytes.truncate(remaining);
1344 }
1345 remaining -= bytes.len();
1346 y.yield_ok(bytes).await;
1347 }
1348 if remaining > 0 {
1349 return Err(incomplete_object_stream_error(remaining));
1350 }
1351 Ok(())
1352 })
1353}
1354
1355fn incomplete_object_stream_error(remaining: impl std::fmt::Display) -> o_Error {
1356 o_Error::Generic {
1357 store: "EcObjectStore",
1358 source: Box::new(std::io::Error::new(
1359 std::io::ErrorKind::UnexpectedEof,
1360 format!("object stream ended with {remaining} bytes remaining"),
1361 )),
1362 }
1363}
1364
1365#[cfg(test)]
1366mod test {
1367 use super::{
1368 EcObjectStore, EcObjectStoreBuildError, OnceCell, SELECT_DEFAULT_READ_BUFFER_SIZE, SelectObjectOptions,
1369 SelectObjectSnapshot, SelectScanRange, SnapshotConsistencyError, bytes_stream, compressed_json_document_ndjson_stream,
1370 convert_csv_delimiter_stream, convert_field_delimiter_stream, convert_record_delimiter_stream,
1371 escaped_json_object_with_size, find_delimiter, flatten_json_document_to_ndjson, http_range_spec_from_get_range,
1372 json_document_ndjson_stream, json_key_eq_ignore_ascii_case_with_checkpoint, legacy_json_source_from_input,
1373 map_storage_error, meter_uncompressed_input_stream, remove_json_source_key_with_checkpoint, scan_range_from_bounds,
1374 scan_range_stream, select_read_headers, snapshot_last_modified, validate_json_document_size,
1375 };
1376 use crate::input_stream::{CompressionFormat, MAX_SELECT_RECORD_BYTES, compressed_input_reader, encode_compressed_fixture};
1377 use crate::query::ast::{JsonPathSegment, JsonSource};
1378 use crate::query::session::{QueryExecutionGuard, QueryExecutionOwner, QueryExecutionTracker};
1379 use crate::storage_api::SelectPutObjReader;
1380 use crate::storage_api::object_store::ObjectIO as _;
1381 use crate::{QueryError, SelectError, SelectInputMetrics, SelectStorageError};
1382 use bytes::Bytes;
1383 use datafusion::{
1384 common::DataFusionError,
1385 execution::memory_pool::{GreedyMemoryPool, MemoryLimit, MemoryPool, MemoryReservation},
1386 execution::{config::SessionConfig, context::SessionContext},
1387 object_store::{self, GetOptions, GetRange, GetResultPayload, ObjectStore as _, path::Path},
1388 physical_plan::ExecutionPlanProperties,
1389 prelude::CsvReadOptions,
1390 };
1391 use futures::{StreamExt, TryStreamExt, stream};
1392 use http::HeaderMap;
1393 use parking_lot::Mutex;
1394 use rustfs_test_utils::PutObjectCommitBarrier;
1395 use s3s::S3ErrorCode;
1396 use s3s::dto::{
1397 CSVInput, CSVOutput, CompressionType, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput,
1398 JSONType, OutputSerialization, ScanRange, SelectObjectContentInput, SelectObjectContentRequest,
1399 };
1400 use s3s::header::{
1401 X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
1402 X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
1403 };
1404 use std::sync::{
1405 Arc,
1406 atomic::{AtomicUsize, Ordering},
1407 };
1408
1409 use tokio::{io::AsyncReadExt, sync::Semaphore};
1410
1411 const JSON_DOCUMENT_TEST_MEMORY_BYTES: usize = 8 * 1024 * 1024;
1412
1413 #[derive(Debug)]
1414 struct AdmissionObservingMemoryPool {
1415 inner: GreedyMemoryPool,
1416 admission: Arc<Semaphore>,
1417 reservation_release: Mutex<Option<tokio::sync::oneshot::Sender<bool>>>,
1418 }
1419
1420 impl AdmissionObservingMemoryPool {
1421 fn new(pool_size: usize, admission: Arc<Semaphore>) -> (Self, tokio::sync::oneshot::Receiver<bool>) {
1422 let (reservation_release, reservation_released) = tokio::sync::oneshot::channel();
1423 (
1424 Self {
1425 inner: GreedyMemoryPool::new(pool_size),
1426 admission,
1427 reservation_release: Mutex::new(Some(reservation_release)),
1428 },
1429 reservation_released,
1430 )
1431 }
1432 }
1433
1434 impl std::fmt::Display for AdmissionObservingMemoryPool {
1435 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1436 std::fmt::Display::fmt(&self.inner, f)
1437 }
1438 }
1439
1440 impl MemoryPool for AdmissionObservingMemoryPool {
1441 fn name(&self) -> &str {
1442 self.inner.name()
1443 }
1444
1445 fn grow(&self, reservation: &MemoryReservation, additional: usize) {
1446 self.inner.grow(reservation, additional);
1447 }
1448
1449 fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
1450 self.inner.shrink(reservation, shrink);
1451 if self.inner.reserved() == 0
1452 && let Some(reservation_release) = self.reservation_release.lock().take()
1453 {
1454 let _ = reservation_release.send(self.admission.available_permits() == 0);
1455 }
1456 }
1457
1458 fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> datafusion::common::Result<()> {
1459 self.inner.try_grow(reservation, additional)
1460 }
1461
1462 fn reserved(&self) -> usize {
1463 self.inner.reserved()
1464 }
1465
1466 fn memory_limit(&self) -> MemoryLimit {
1467 self.inner.memory_limit()
1468 }
1469 }
1470
1471 fn source_key(name: &str) -> JsonPathSegment {
1472 JsonPathSegment::Key {
1473 name: name.to_string(),
1474 quoted: false,
1475 }
1476 }
1477
1478 fn quoted_source_key(name: &str) -> JsonPathSegment {
1479 JsonPathSegment::Key {
1480 name: name.to_string(),
1481 quoted: true,
1482 }
1483 }
1484
1485 fn csv_input(bucket: &str, object: &str) -> Arc<SelectObjectContentInput> {
1486 Arc::new(SelectObjectContentInput {
1487 bucket: bucket.to_string(),
1488 expected_bucket_owner: None,
1489 key: object.to_string(),
1490 sse_customer_algorithm: None,
1491 sse_customer_key: None,
1492 sse_customer_key_md5: None,
1493 request: SelectObjectContentRequest {
1494 expression: "SELECT * FROM s3object".to_string(),
1495 expression_type: ExpressionType::from_static(ExpressionType::SQL),
1496 input_serialization: InputSerialization {
1497 csv: Some(CSVInput::default()),
1498 ..Default::default()
1499 },
1500 output_serialization: OutputSerialization {
1501 csv: Some(CSVOutput::default()),
1502 ..Default::default()
1503 },
1504 request_progress: None,
1505 scan_range: None,
1506 },
1507 })
1508 }
1509
1510 fn json_document_input(bucket: &str, object: &str, expression: &str) -> Arc<SelectObjectContentInput> {
1511 Arc::new(SelectObjectContentInput {
1512 bucket: bucket.to_string(),
1513 expected_bucket_owner: None,
1514 key: object.to_string(),
1515 sse_customer_algorithm: None,
1516 sse_customer_key: None,
1517 sse_customer_key_md5: None,
1518 request: SelectObjectContentRequest {
1519 expression: expression.to_string(),
1520 expression_type: ExpressionType::from_static(ExpressionType::SQL),
1521 input_serialization: InputSerialization {
1522 json: Some(JSONInput {
1523 type_: Some(JSONType::from_static(JSONType::DOCUMENT)),
1524 }),
1525 ..Default::default()
1526 },
1527 output_serialization: OutputSerialization {
1528 csv: Some(CSVOutput::default()),
1529 ..Default::default()
1530 },
1531 request_progress: None,
1532 scan_range: None,
1533 },
1534 })
1535 }
1536
1537 #[test]
1538 fn legacy_source_adapter_normalizes_scalar_bindings() {
1539 let input = json_document_input("bucket", "input.json", "SELECT V FROM S3Object AS V");
1540 let implicit = json_document_input("bucket", "input.json", "SELECT S3Object FROM S3Object");
1541 let wildcard = json_document_input("bucket", "input.json", "SELECT _1 FROM S3Object[*]");
1542 let key = json_document_input("bucket", "input.json", "SELECT * FROM S3Object.LongKey");
1543 let quoted_key = json_document_input("bucket", "input.json", "SELECT * FROM S3Object.\"LongKey\"");
1544
1545 assert_eq!(legacy_json_source_from_input(&input), JsonSource::new(Vec::new(), Some("v".to_string())));
1546 assert_eq!(
1547 legacy_json_source_from_input(&implicit),
1548 JsonSource::new(Vec::new(), Some("s3object".to_string()))
1549 );
1550 assert_eq!(
1551 legacy_json_source_from_input(&wildcard),
1552 JsonSource::new(vec![JsonPathSegment::ArrayWildcard], Some("_1".to_string()))
1553 );
1554 assert_eq!(legacy_json_source_from_input(&key).scalar_column(), Some("longkey"));
1555 assert_eq!(legacy_json_source_from_input("ed_key).scalar_column(), Some("LongKey"));
1556 }
1557
1558 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1559 #[serial_test::serial]
1560 async fn legacy_public_constructors_preserve_single_key_json_source() {
1561 const BUCKET: &str = "s3select-legacy-json-source";
1562 const OBJECT: &str = "input.json";
1563
1564 let env = crate::storage_api::select_test_ecstore_env().await;
1565 env.make_bucket(BUCKET, false).await;
1566 env.put_object_bytes(BUCKET, OBJECT, br#"{"employees":[{"name":"Alice"}]}"#.to_vec())
1567 .await;
1568 let input = json_document_input(BUCKET, OBJECT, "SELECT e.name FROM S3Object.employees AS e");
1569 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
1570
1571 let lazy = EcObjectStore::new(Arc::clone(&input)).expect("legacy lazy constructor should resolve the global store");
1572 let pinned =
1573 EcObjectStore::new_with_snapshot(input, snapshot).expect("legacy pinned constructor should accept the snapshot");
1574 let expected = JsonSource::new(vec![source_key("employees")], Some("e".to_string()));
1575
1576 assert_eq!(lazy.json_source, expected);
1577 assert_eq!(pinned.json_source, expected);
1578 }
1579
1580 fn json_input(bucket: &str, object: &str, json_type: &'static str) -> Arc<SelectObjectContentInput> {
1581 let mut input = (*csv_input(bucket, object)).clone();
1582 input.request.input_serialization = InputSerialization {
1583 json: Some(JSONInput {
1584 type_: Some(JSONType::from_static(json_type)),
1585 }),
1586 ..Default::default()
1587 };
1588 input.request.output_serialization = OutputSerialization {
1589 json: Some(JSONOutput::default()),
1590 ..Default::default()
1591 };
1592 Arc::new(input)
1593 }
1594
1595 #[test]
1596 fn lazy_snapshot_headers_preserve_ssec_context() {
1597 let mut input = (*csv_input("bucket", "object.csv")).clone();
1598 input.sse_customer_algorithm = Some("AES256".to_string());
1599 input.sse_customer_key = Some("customer-key".to_string());
1600 input.sse_customer_key_md5 = Some("customer-key-md5".to_string());
1601
1602 let headers = select_read_headers(&input);
1603
1604 assert_eq!(
1605 headers
1606 .get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM)
1607 .and_then(|value| value.to_str().ok()),
1608 Some("AES256")
1609 );
1610 assert_eq!(
1611 headers
1612 .get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY)
1613 .and_then(|value| value.to_str().ok()),
1614 Some("customer-key")
1615 );
1616 assert_eq!(
1617 headers
1618 .get(X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5)
1619 .and_then(|value| value.to_str().ok()),
1620 Some("customer-key-md5")
1621 );
1622 }
1623
1624 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1625 #[serial_test::serial]
1626 async fn legacy_constructor_retries_snapshot_after_not_found() {
1627 let env = crate::storage_api::select_test_ecstore_env().await;
1628 let bucket = "s3select-lazy-snapshot-retry";
1629 let object = "input.csv";
1630 env.make_bucket(bucket, false).await;
1631 let store = EcObjectStore::new(csv_input(bucket, object)).expect("legacy constructor should resolve the global store");
1632
1633 let error = store
1634 .get_opts(
1635 &Path::from(object),
1636 GetOptions {
1637 head: true,
1638 ..Default::default()
1639 },
1640 )
1641 .await
1642 .expect_err("missing object should remain a typed not-found error");
1643 assert!(matches!(error, object_store::Error::NotFound { .. }));
1644
1645 env.put_object_bytes(bucket, object, b"id,name\n1,Alice\n".to_vec()).await;
1646 let result = store
1647 .get_opts(
1648 &Path::from(object),
1649 GetOptions {
1650 head: true,
1651 ..Default::default()
1652 },
1653 )
1654 .await
1655 .expect("failed snapshot initialization must not be cached");
1656 assert_eq!(result.meta.size, 16);
1657 }
1658
1659 #[tokio::test]
1660 #[serial_test::serial]
1661 async fn legacy_constructor_maps_missing_bucket_to_not_found() {
1662 let _env = crate::storage_api::select_test_ecstore_env().await;
1663 let bucket = "s3select-lazy-snapshot-missing-bucket";
1664 let object = "input.csv";
1665 let store = EcObjectStore::new(csv_input(bucket, object)).expect("legacy constructor should resolve the global store");
1666
1667 let error = store
1668 .get_opts(
1669 &Path::from(object),
1670 GetOptions {
1671 head: true,
1672 ..Default::default()
1673 },
1674 )
1675 .await
1676 .expect_err("missing bucket should remain a typed not-found error");
1677
1678 assert!(matches!(error, object_store::Error::NotFound { .. }));
1679 }
1680
1681 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1682 #[serial_test::serial]
1683 async fn legacy_constructor_reuses_head_snapshot_for_body() {
1684 const BUCKET: &str = "s3select-lazy-snapshot-head-body";
1685 const OBJECT: &str = "input.csv";
1686 const OLD_DATA: &[u8] = b"id,name\n1,old\n";
1687 const NEW_DATA: &[u8] = b"id,name\n1,new\n";
1688
1689 let env = crate::storage_api::select_test_ecstore_env().await;
1690 env.make_bucket(BUCKET, false).await;
1691 env.put_object_bytes(BUCKET, OBJECT, OLD_DATA.to_vec()).await;
1692 let store = EcObjectStore::new(csv_input(BUCKET, OBJECT)).expect("legacy constructor should resolve the global store");
1693 let head = store
1694 .get_opts(
1695 &Path::from(OBJECT),
1696 GetOptions {
1697 head: true,
1698 ..Default::default()
1699 },
1700 )
1701 .await
1702 .expect("HEAD should lazily prepare the snapshot");
1703 assert_eq!(head.meta.size, u64::try_from(OLD_DATA.len()).expect("fixture length should fit in u64"));
1704
1705 let commit_barrier = PutObjectCommitBarrier::before_namespace(BUCKET, OBJECT);
1706 let writer = tokio::spawn(async move {
1707 env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()).await;
1708 });
1709 commit_barrier.wait_until_paused().await;
1710 commit_barrier.release_and_wait_until_namespace_pending().await;
1711 assert!(!writer.is_finished(), "overwrite must wait for the lazy snapshot");
1712
1713 let result = store
1714 .get_opts(&Path::from(OBJECT), GetOptions::default())
1715 .await
1716 .expect("body should reuse the HEAD snapshot");
1717 let GetResultPayload::Stream(stream) = result.payload else {
1718 panic!("expected streaming snapshot body");
1719 };
1720 let bytes = stream
1721 .try_collect::<Vec<_>>()
1722 .await
1723 .expect("collect lazy snapshot body")
1724 .concat();
1725 assert_eq!(bytes, OLD_DATA);
1726 assert_eq!(store.reader_open_count.load(Ordering::Relaxed), 1);
1727
1728 drop(store);
1729 tokio::time::timeout(std::time::Duration::from_secs(5), writer)
1730 .await
1731 .expect("overwrite should finish after the lazy snapshot is released")
1732 .expect("overwrite task should join");
1733 assert_eq!(read_current_object(BUCKET, OBJECT).await, NEW_DATA);
1734 }
1735
1736 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1737 #[serial_test::serial]
1738 async fn legacy_constructor_first_version_pins_later_reads() {
1739 const BUCKET: &str = "s3select-lazy-snapshot-version";
1740 const OBJECT: &str = "input.csv";
1741 const OLD_DATA: &[u8] = b"old-marker\n";
1742 const NEW_DATA: &[u8] = b"new-poison-value\n";
1743
1744 let env = crate::storage_api::select_test_ecstore_env().await;
1745 env.make_bucket(BUCKET, true).await;
1746 let versioned_opts = SelectObjectOptions {
1747 versioned: true,
1748 ..Default::default()
1749 };
1750 let mut old_reader = SelectPutObjReader::from_vec(OLD_DATA.to_vec());
1751 let old_info = env
1752 .ecstore
1753 .put_object(BUCKET, OBJECT, &mut old_reader, &versioned_opts)
1754 .await
1755 .expect("put old version fixture");
1756 let old_version = old_info
1757 .version_id
1758 .expect("versioned PUT should return a version ID")
1759 .to_string();
1760 let mut new_reader = SelectPutObjReader::from_vec(NEW_DATA.to_vec());
1761 let new_info = env
1762 .ecstore
1763 .put_object(BUCKET, OBJECT, &mut new_reader, &versioned_opts)
1764 .await
1765 .expect("put latest version poison fixture");
1766 let new_version = new_info
1767 .version_id
1768 .expect("versioned PUT should return a version ID")
1769 .to_string();
1770
1771 let store = EcObjectStore::new(csv_input(BUCKET, OBJECT)).expect("legacy constructor should resolve the global store");
1772 let head = store
1773 .get_opts(
1774 &Path::from(OBJECT),
1775 GetOptions {
1776 head: true,
1777 version: Some(old_version.to_uppercase()),
1778 ..Default::default()
1779 },
1780 )
1781 .await
1782 .expect("first HEAD should bind the requested old version");
1783 assert_eq!(head.meta.version.as_deref(), Some(old_version.as_str()));
1784
1785 let mismatch = store
1786 .get_opts(
1787 &Path::from(OBJECT),
1788 GetOptions {
1789 head: true,
1790 version: Some(new_version),
1791 ..Default::default()
1792 },
1793 )
1794 .await
1795 .expect_err("an explicit different version must not reuse the pinned snapshot");
1796 assert!(mismatch.to_string().contains("different object version"));
1797
1798 let range = store
1799 .get_opts(
1800 &Path::from(OBJECT),
1801 GetOptions {
1802 range: Some(GetRange::Bounded(0..3)),
1803 ..Default::default()
1804 },
1805 )
1806 .await
1807 .expect("later range should reuse the old-version snapshot");
1808 let GetResultPayload::Stream(range_stream) = range.payload else {
1809 panic!("expected ranged snapshot stream");
1810 };
1811 assert_eq!(
1812 range_stream
1813 .try_collect::<Vec<_>>()
1814 .await
1815 .expect("collect old-version range")
1816 .concat(),
1817 b"old"
1818 );
1819
1820 let body = store
1821 .get_opts(&Path::from(OBJECT), GetOptions::default())
1822 .await
1823 .expect("later body should reuse the old-version snapshot");
1824 let GetResultPayload::Stream(body_stream) = body.payload else {
1825 panic!("expected full snapshot stream");
1826 };
1827 assert_eq!(
1828 body_stream
1829 .try_collect::<Vec<_>>()
1830 .await
1831 .expect("collect old-version body")
1832 .concat(),
1833 OLD_DATA
1834 );
1835 }
1836
1837 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1838 #[serial_test::serial]
1839 async fn legacy_constructor_normalizes_null_version_before_snapshot_prepare() {
1840 const BUCKET: &str = "s3select-lazy-snapshot-null-version";
1841 const OBJECT: &str = "input.csv";
1842 const DATA: &[u8] = b"null-version-marker\n";
1843
1844 let env = crate::storage_api::select_test_ecstore_env().await;
1845 env.make_bucket(BUCKET, false).await;
1846 env.put_object_bytes(BUCKET, OBJECT, DATA.to_vec()).await;
1847 let store = EcObjectStore::new(csv_input(BUCKET, OBJECT)).expect("legacy constructor should resolve the global store");
1848
1849 let head = store
1850 .get_opts(
1851 &Path::from(OBJECT),
1852 GetOptions {
1853 head: true,
1854 version: Some("NULL".to_string()),
1855 ..Default::default()
1856 },
1857 )
1858 .await
1859 .expect("null version should prepare an unversioned snapshot");
1860 assert!(head.meta.version.is_none());
1861
1862 let body = store
1863 .get_opts(
1864 &Path::from(OBJECT),
1865 GetOptions {
1866 version: Some(uuid::Uuid::nil().to_string()),
1867 ..Default::default()
1868 },
1869 )
1870 .await
1871 .expect("nil UUID should match the pinned null-version snapshot");
1872 let GetResultPayload::Stream(body_stream) = body.payload else {
1873 panic!("expected null-version snapshot stream");
1874 };
1875 assert_eq!(
1876 body_stream
1877 .try_collect::<Vec<_>>()
1878 .await
1879 .expect("collect null-version body")
1880 .concat(),
1881 DATA
1882 );
1883 }
1884
1885 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1886 #[serial_test::serial]
1887 async fn prepared_snapshot_rejects_a_different_query_object() {
1888 const BUCKET: &str = "s3select-snapshot-identity";
1889 const OBJECT: &str = "source.csv";
1890
1891 let env = crate::storage_api::select_test_ecstore_env().await;
1892 env.make_bucket(BUCKET, false).await;
1893 env.put_object_bytes(BUCKET, OBJECT, b"source-marker\n".to_vec()).await;
1894 let snapshot = Arc::new(
1895 env.ecstore
1896 .prepare_select_object_snapshot(BUCKET, OBJECT, &HeaderMap::new(), &Default::default())
1897 .await
1898 .expect("prepare source snapshot"),
1899 );
1900
1901 let error = EcObjectStore::new_with_snapshot(csv_input(BUCKET, "different.csv"), snapshot)
1902 .expect_err("a snapshot must remain bound to its source object");
1903
1904 assert_eq!(error.code(), &S3ErrorCode::InternalError);
1905 assert!(error.source().is_some_and(|source| {
1906 source
1907 .downcast_ref::<EcObjectStoreBuildError>()
1908 .is_some_and(|error| matches!(error, EcObjectStoreBuildError::Snapshot(SnapshotConsistencyError::ObjectChanged)))
1909 }));
1910 }
1911
1912 async fn prepare_test_snapshot(bucket: &str, object: &str) -> Arc<SelectObjectSnapshot> {
1913 let env = crate::storage_api::select_test_ecstore_env().await;
1914 Arc::new(
1915 env.ecstore
1916 .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &Default::default())
1917 .await
1918 .expect("prepare SelectObjectContent snapshot"),
1919 )
1920 }
1921
1922 fn scan_range_csv_store(
1923 bucket: &str,
1924 object: &str,
1925 snapshot: Arc<SelectObjectSnapshot>,
1926 record_delimiter: &str,
1927 file_header_info: Option<FileHeaderInfo>,
1928 start: i64,
1929 end: i64,
1930 ) -> EcObjectStore {
1931 EcObjectStore::new_with_snapshot(
1932 Arc::new(SelectObjectContentInput {
1933 bucket: bucket.to_string(),
1934 expected_bucket_owner: None,
1935 key: object.to_string(),
1936 sse_customer_algorithm: None,
1937 sse_customer_key: None,
1938 sse_customer_key_md5: None,
1939 request: SelectObjectContentRequest {
1940 expression: "SELECT * FROM s3object".to_string(),
1941 expression_type: ExpressionType::from_static(ExpressionType::SQL),
1942 input_serialization: InputSerialization {
1943 csv: Some(CSVInput {
1944 record_delimiter: Some(record_delimiter.to_string()),
1945 file_header_info,
1946 ..Default::default()
1947 }),
1948 ..Default::default()
1949 },
1950 output_serialization: OutputSerialization {
1951 csv: Some(CSVOutput::default()),
1952 ..Default::default()
1953 },
1954 request_progress: None,
1955 scan_range: Some(ScanRange {
1956 start: Some(start),
1957 end: Some(end),
1958 }),
1959 },
1960 }),
1961 snapshot,
1962 )
1963 .expect("snapshot should match SelectObjectContent input")
1964 }
1965
1966 async fn read_current_object(bucket: &str, object: &str) -> Vec<u8> {
1967 let snapshot = prepare_test_snapshot(bucket, object).await;
1968 let mut reader = snapshot.open_reader(None).await.expect("current object reader should open");
1969 let mut bytes = Vec::new();
1970 reader
1971 .stream
1972 .read_to_end(&mut bytes)
1973 .await
1974 .expect("current object should be readable");
1975 bytes
1976 }
1977
1978 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1979 #[serial_test::serial]
1980 async fn head_uses_snapshot_metadata_without_opening_body() {
1981 let env = crate::storage_api::select_test_ecstore_env().await;
1982 let bucket = "s3select-snapshot-head";
1983 let object = "input.csv";
1984 env.make_bucket(bucket, false).await;
1985 let mut reader = SelectPutObjReader::from_vec(b"id,name\n1,Alice\n".to_vec());
1986 env.ecstore
1987 .put_object(bucket, object, &mut reader, &Default::default())
1988 .await
1989 .expect("put HEAD fixture");
1990
1991 let snapshot = prepare_test_snapshot(bucket, object).await;
1992 let expected_modified = snapshot_last_modified(&snapshot).expect("snapshot modification time");
1993 let expected_size = snapshot.logical_size();
1994 let expected_etag = snapshot.object_info().etag.clone();
1995 let expected_version = snapshot.object_info().version_id.map(|version| version.to_string());
1996 let input = Arc::new(SelectObjectContentInput {
1997 bucket: bucket.to_string(),
1998 expected_bucket_owner: None,
1999 key: object.to_string(),
2000 sse_customer_algorithm: Some("secret-algorithm".to_string()),
2001 sse_customer_key: Some("secret-customer-key".to_string()),
2002 sse_customer_key_md5: Some("secret-customer-key-md5".to_string()),
2003 request: SelectObjectContentRequest {
2004 expression: "SELECT * FROM s3object".to_string(),
2005 expression_type: ExpressionType::from_static(ExpressionType::SQL),
2006 input_serialization: InputSerialization {
2007 csv: Some(CSVInput::default()),
2008 ..Default::default()
2009 },
2010 output_serialization: OutputSerialization {
2011 csv: Some(CSVOutput::default()),
2012 ..Default::default()
2013 },
2014 request_progress: None,
2015 scan_range: None,
2016 },
2017 });
2018 let store = EcObjectStore::new_with_snapshot(input, snapshot).expect("snapshot should match SelectObjectContent input");
2019 let debug = format!("{store:?}");
2020 assert!(!debug.contains("secret-customer-key"));
2021
2022 let result = store
2023 .get_opts(
2024 &Path::from(object),
2025 GetOptions {
2026 head: true,
2027 ..Default::default()
2028 },
2029 )
2030 .await
2031 .expect("HEAD from snapshot metadata");
2032
2033 assert_eq!(result.meta.last_modified, expected_modified);
2034 assert_eq!(result.meta.size, expected_size);
2035 assert_eq!(result.meta.e_tag, expected_etag);
2036 assert_eq!(result.meta.version, expected_version);
2037 assert_eq!(store.reader_open_count.load(Ordering::Relaxed), 0);
2038 }
2039
2040 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2041 #[serial_test::serial]
2042 async fn snapshot_keeps_csv_header_and_body_on_one_generation_during_overwrite() {
2043 const BUCKET: &str = "s3select-snapshot-header-body-race";
2044 const OBJECT: &str = "input.csv";
2045 const OLD_DATA: &[u8] = b"old_header,value\nskip_old,0\nold_body,1\n";
2046 const NEW_DATA: &[u8] = b"new_header,value\nskip_new,0\nnew_body,1\n";
2047
2048 let env = crate::storage_api::select_test_ecstore_env().await;
2049 env.make_bucket(BUCKET, false).await;
2050 let mut reader = SelectPutObjReader::from_vec(OLD_DATA.to_vec());
2051 env.ecstore
2052 .put_object(BUCKET, OBJECT, &mut reader, &Default::default())
2053 .await
2054 .expect("put old CSV header/body fixture");
2055
2056 let selected_start = i64::try_from(b"old_header,value\nskip_old,0\n".len()).expect("fixture offset should fit in i64");
2057 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
2058 let store = scan_range_csv_store(
2059 BUCKET,
2060 OBJECT,
2061 snapshot,
2062 "\n",
2063 Some(FileHeaderInfo::from_static(FileHeaderInfo::USE)),
2064 selected_start,
2065 selected_start,
2066 );
2067 let commit_barrier = PutObjectCommitBarrier::before_namespace(BUCKET, OBJECT);
2068 let writer = tokio::spawn(async move {
2069 env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()).await;
2070 });
2071 commit_barrier.wait_until_paused().await;
2072 commit_barrier.release_and_wait_until_namespace_pending().await;
2073 assert!(
2074 !writer.is_finished(),
2075 "overwrite must remain blocked while the SelectObjectContent snapshot is alive"
2076 );
2077
2078 let result = store
2079 .get_opts(&Path::from(OBJECT), GetOptions::default())
2080 .await
2081 .expect("read CSV header and body from one snapshot");
2082 let GetResultPayload::Stream(stream) = result.payload else {
2083 panic!("expected streaming CSV header/body payload");
2084 };
2085 let bytes = stream
2086 .try_collect::<Vec<_>>()
2087 .await
2088 .expect("collect CSV header/body snapshot")
2089 .concat();
2090
2091 assert_eq!(bytes, b"old_header,value\nold_body,1\n");
2092 assert_eq!(store.reader_open_count.load(Ordering::Relaxed), 2);
2093 assert!(!writer.is_finished(), "overwrite must remain blocked after both snapshot readers finish");
2094
2095 drop(store);
2096 tokio::time::timeout(std::time::Duration::from_secs(5), writer)
2097 .await
2098 .expect("overwrite should finish after the snapshot is released")
2099 .expect("overwrite task should join");
2100 assert_eq!(read_current_object(BUCKET, OBJECT).await, NEW_DATA);
2101 }
2102
2103 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2104 #[serial_test::serial]
2105 async fn snapshot_keeps_scan_range_context_and_main_reader_on_one_generation_during_overwrite() {
2106 const BUCKET: &str = "s3select-snapshot-scan-context-race";
2107 const OBJECT: &str = "input.csv";
2108 const OLD_DATA: &[u8] = b"111aaa222aa333aa";
2109 const NEW_DATA: &[u8] = b"999aaa888aa777aa";
2110
2111 let env = crate::storage_api::select_test_ecstore_env().await;
2112 env.make_bucket(BUCKET, false).await;
2113 let mut reader = SelectPutObjReader::from_vec(OLD_DATA.to_vec());
2114 env.ecstore
2115 .put_object(BUCKET, OBJECT, &mut reader, &Default::default())
2116 .await
2117 .expect("put old ScanRange context fixture");
2118
2119 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
2120 let store = scan_range_csv_store(BUCKET, OBJECT, snapshot, "aa", None, 4, 5);
2121 let commit_barrier = PutObjectCommitBarrier::before_namespace(BUCKET, OBJECT);
2122 let writer = tokio::spawn(async move {
2123 env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()).await;
2124 });
2125 commit_barrier.wait_until_paused().await;
2126 commit_barrier.release_and_wait_until_namespace_pending().await;
2127 assert!(
2128 !writer.is_finished(),
2129 "overwrite must remain blocked while the SelectObjectContent snapshot is alive"
2130 );
2131
2132 let result = store
2133 .get_opts(&Path::from(OBJECT), GetOptions::default())
2134 .await
2135 .expect("read ScanRange context and main body from one snapshot");
2136 let GetResultPayload::Stream(stream) = result.payload else {
2137 panic!("expected streaming ScanRange context payload");
2138 };
2139 let bytes = stream
2140 .try_collect::<Vec<_>>()
2141 .await
2142 .expect("collect ScanRange context snapshot")
2143 .concat();
2144
2145 assert_eq!(bytes, b"a222\r\n");
2146 assert_eq!(store.reader_open_count.load(Ordering::Relaxed), 2);
2147 assert!(
2148 !writer.is_finished(),
2149 "overwrite must remain blocked after context and main readers finish"
2150 );
2151
2152 drop(store);
2153 tokio::time::timeout(std::time::Duration::from_secs(5), writer)
2154 .await
2155 .expect("overwrite should finish after the snapshot is released")
2156 .expect("overwrite task should join");
2157 assert_eq!(read_current_object(BUCKET, OBJECT).await, NEW_DATA);
2158 }
2159
2160 #[tokio::test]
2161 async fn test_scan_range_stream_keeps_header_and_selected_record() {
2162 let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"h1,h2\n1,a\n2,b\n3,c\n"))]);
2163 let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange::new(10, 11), true, 0, 18);
2164 let mut output = Vec::new();
2165 while let Some(bytes) = stream.next().await {
2166 output.extend_from_slice(&bytes.unwrap());
2167 }
2168 assert_eq!(output, b"h1,h2\n2,b\n");
2169 }
2170
2171 #[tokio::test]
2172 async fn test_scan_range_stream_skips_record_when_start_is_in_middle() {
2173 let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"1,a\n2,b\n3,c\n"))]);
2174 let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange::new(2, 7), false, 0, 12);
2175 let mut output = Vec::new();
2176 while let Some(bytes) = stream.next().await {
2177 output.extend_from_slice(&bytes.unwrap());
2178 }
2179 assert_eq!(output, b"2,b\n");
2180 }
2181
2182 #[tokio::test]
2183 async fn test_scan_range_stream_keeps_record_when_end_is_in_middle() {
2184 let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"1,a\n2,b\n3,c\n"))]);
2185 let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange::new(0, 5), false, 0, 12);
2186 let mut output = Vec::new();
2187 while let Some(bytes) = stream.next().await {
2188 output.extend_from_slice(&bytes.unwrap());
2189 }
2190 assert_eq!(output, b"1,a\n2,b\n");
2191 }
2192
2193 #[tokio::test]
2194 async fn test_scan_range_stream_uses_base_offset_for_range_reader() {
2195 let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"\n2,b\n3,c\n"))]);
2196 let mut stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange::new(4, 7), false, 3, 12);
2197 let mut output = Vec::new();
2198 while let Some(bytes) = stream.next().await {
2199 output.extend_from_slice(&bytes.unwrap());
2200 }
2201 assert_eq!(output, b"2,b\n");
2202 }
2203
2204 #[tokio::test]
2205 async fn test_scan_range_stream_handles_delimiter_split_across_chunks() {
2206 let chunks = stream::iter(vec![
2207 Ok::<_, std::io::Error>(Bytes::from_static(b"h1,h2\r")),
2208 Ok::<_, std::io::Error>(Bytes::from_static(b"\n1,a\r\n2,b\r")),
2209 Ok::<_, std::io::Error>(Bytes::from_static(b"\n3,c\r\n")),
2210 ]);
2211 let mut stream = scan_range_stream(chunks, b"\r\n".to_vec(), SelectScanRange::new(12, 14), true, 0, 22);
2212 let mut output = Vec::new();
2213 while let Some(bytes) = stream.next().await {
2214 output.extend_from_slice(&bytes.unwrap());
2215 }
2216 assert_eq!(output, b"h1,h2\r\n2,b\r\n");
2217 }
2218
2219 #[tokio::test]
2220 async fn test_scan_range_stream_handles_many_records_in_one_chunk() {
2221 const RECORDS: usize = 16 * 1024;
2222 let input = b"x\n".repeat(RECORDS);
2223 let object_size = u64::try_from(input.len()).expect("fixture length should fit in u64");
2224 let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from(input.clone()))]);
2225 let output = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange::new(0, object_size - 1), false, 0, object_size)
2226 .try_collect::<Vec<_>>()
2227 .await
2228 .expect("collect many ScanRange records")
2229 .concat();
2230
2231 assert_eq!(output, input);
2232 }
2233
2234 #[tokio::test]
2235 async fn test_scan_range_stream_matches_reference_across_chunk_boundaries() {
2236 fn reference(input: &[u8], delimiter: &[u8], range: SelectScanRange, include_header: bool) -> Vec<u8> {
2237 let mut output = Vec::new();
2238 let mut record_begin = 0;
2239 let mut record_start = 0_u64;
2240 loop {
2241 let record_end = input[record_begin..]
2242 .windows(delimiter.len())
2243 .position(|window| window == delimiter)
2244 .map(|position| record_begin + position + delimiter.len())
2245 .unwrap_or(input.len());
2246 let include_record = record_start >= range.start() && record_start <= range.end();
2247 if (include_header && record_start == 0) || include_record {
2248 output.extend_from_slice(&input[record_begin..record_end]);
2249 } else if record_start > range.end() {
2250 break;
2251 }
2252 if record_end == input.len() {
2253 break;
2254 }
2255 record_start = record_start.saturating_add(u64::try_from(record_end - record_begin).unwrap_or(u64::MAX));
2256 record_begin = record_end;
2257 }
2258 output
2259 }
2260
2261 let fixtures: &[(&[u8], &[u8])] = &[
2262 (b"header\none\ntwo\nthree", b"\n"),
2263 (b"header\r\none\r\ntwo\r\nthree", b"\r\n"),
2264 (b"headeraaoneaaaxaatwo", b"aa"),
2265 ];
2266 for (input, delimiter) in fixtures {
2267 let object_size = u64::try_from(input.len()).expect("fixture length should fit in u64");
2268 let ranges = [
2269 SelectScanRange::new(0, object_size - 1),
2270 SelectScanRange::new(1, object_size / 2),
2271 SelectScanRange::new(object_size / 2, object_size - 1),
2272 ];
2273 for range in ranges {
2274 for include_header in [false, true] {
2275 let expected = reference(input, delimiter, range, include_header);
2276 for chunk_size in 1..=input.len() {
2277 let chunks = input
2278 .chunks(chunk_size)
2279 .map(|chunk| Ok::<_, std::io::Error>(Bytes::copy_from_slice(chunk)))
2280 .collect::<Vec<_>>();
2281 let actual =
2282 scan_range_stream(stream::iter(chunks), delimiter.to_vec(), range, include_header, 0, object_size)
2283 .try_collect::<Vec<_>>()
2284 .await
2285 .expect("collect ScanRange output")
2286 .concat();
2287
2288 assert_eq!(actual, expected, "delimiter={delimiter:?}, range={range:?}, chunk_size={chunk_size}");
2289 }
2290 }
2291 }
2292 }
2293 }
2294
2295 #[tokio::test]
2296 async fn test_scan_range_stream_converts_custom_delimiter_split_across_chunks() {
2297 let chunks = stream::iter(vec![
2298 Ok::<_, std::io::Error>(Bytes::from_static(b"h1,h2^")),
2299 Ok::<_, std::io::Error>(Bytes::from_static(b"Y1,a^Y2,b^")),
2300 Ok::<_, std::io::Error>(Bytes::from_static(b"Y3,c^Y")),
2301 ]);
2302 let stream = scan_range_stream(chunks, b"^Y".to_vec(), SelectScanRange::new(12, 14), true, 0, 22);
2303 let mut stream = convert_record_delimiter_stream(stream, b"^Y".to_vec());
2304 let mut output = Vec::new();
2305 while let Some(bytes) = stream.next().await {
2306 output.extend_from_slice(&bytes.expect("custom-delimiter ScanRange chunk should be valid"));
2307 }
2308 assert_eq!(output, b"h1,h2\r\n2,b\r\n");
2309 }
2310
2311 #[tokio::test]
2312 async fn test_record_delimiter_conversion_carries_partial_delimiter() {
2313 let chunks = stream::iter(vec![
2314 Ok::<_, object_store::Error>(Bytes::from_static(b"a,1^")),
2315 Ok::<_, object_store::Error>(Bytes::from_static(b"Yb,2^Y")),
2316 ]);
2317 let output = convert_record_delimiter_stream(chunks, b"^Y".to_vec())
2318 .try_collect::<Vec<_>>()
2319 .await
2320 .expect("convert record delimiter")
2321 .concat();
2322 assert_eq!(output, b"a,1\r\nb,2\r\n");
2323 }
2324
2325 #[tokio::test]
2326 async fn test_record_delimiter_conversion_preserves_overlapping_match_order() {
2327 let chunks = stream::iter(vec![
2328 Ok::<_, object_store::Error>(Bytes::from_static(b"a")),
2329 Ok::<_, object_store::Error>(Bytes::from_static(b"aa")),
2330 Ok::<_, object_store::Error>(Bytes::from_static(b"a")),
2331 ]);
2332 let output = convert_record_delimiter_stream(chunks, b"aa".to_vec())
2333 .try_collect::<Vec<_>>()
2334 .await
2335 .expect("convert overlapping record delimiter")
2336 .concat();
2337 assert_eq!(output, b"\r\n\r\n");
2338 }
2339
2340 #[tokio::test]
2341 async fn test_record_and_field_delimiter_conversion_order() {
2342 let chunks = stream::iter(vec![
2343 Ok::<_, object_store::Error>(Bytes::from_static(b"a\r")),
2344 Ok::<_, object_store::Error>(Bytes::from_static(b"\n1^")),
2345 Ok::<_, object_store::Error>(Bytes::from_static(b"Yb\r\n2^Y")),
2346 ]);
2347 let output = convert_csv_delimiter_stream(chunks, Some(b"^Y".to_vec()), Some("\r\n".to_string()))
2348 .try_collect::<Vec<_>>()
2349 .await
2350 .expect("convert record and field delimiters")
2351 .concat();
2352 assert_eq!(output, b"a,1\r\nb,2\r\n");
2353 }
2354
2355 #[tokio::test]
2356 async fn test_record_delimiter_takes_precedence_when_delimiters_match() {
2357 let chunks = stream::iter(vec![
2358 Ok::<_, object_store::Error>(Bytes::from_static(b"a^")),
2359 Ok::<_, object_store::Error>(Bytes::from_static(b"Yb^Y")),
2360 ]);
2361 let output = convert_csv_delimiter_stream(chunks, Some(b"^Y".to_vec()), Some("^Y".to_string()))
2362 .try_collect::<Vec<_>>()
2363 .await
2364 .expect("convert matching record and field delimiters")
2365 .concat();
2366 assert_eq!(output, b"a\r\nb\r\n");
2367
2368 let chunks = stream::iter(vec![Ok::<_, object_store::Error>(Bytes::from_static(b"a\r\nb\r\n"))]);
2369 let output = convert_csv_delimiter_stream(chunks, Some(b"\r\n".to_vec()), Some("\r\n".to_string()))
2370 .try_collect::<Vec<_>>()
2371 .await
2372 .expect("preserve matching normalized record delimiter")
2373 .concat();
2374 assert_eq!(output, b"a\r\nb\r\n");
2375 }
2376
2377 #[tokio::test]
2378 async fn test_longer_field_delimiter_takes_precedence_over_record_prefix() {
2379 let chunks = stream::iter(vec![
2380 Ok::<_, object_store::Error>(Bytes::from_static(b"a^")),
2381 Ok::<_, object_store::Error>(Bytes::from_static(b"YQb^Y")),
2382 ]);
2383 let output = convert_csv_delimiter_stream(chunks, Some(b"^Y".to_vec()), Some("^YQ".to_string()))
2384 .try_collect::<Vec<_>>()
2385 .await
2386 .expect("convert record delimiter that prefixes field delimiter")
2387 .concat();
2388 assert_eq!(output, b"a,b\r\n");
2389
2390 let chunks = stream::iter(vec![Ok::<_, object_store::Error>(Bytes::from_static(b"a\nXb\nc\nXd\n"))]);
2391 let output = convert_csv_delimiter_stream(chunks, Some(b"\n".to_vec()), Some("\nX".to_string()))
2392 .try_collect::<Vec<_>>()
2393 .await
2394 .expect("preserve longer field delimiter with native record prefix")
2395 .concat();
2396 assert_eq!(output, b"a,b\nc,d\n");
2397 }
2398
2399 #[tokio::test]
2400 async fn test_scan_range_stream_rejects_early_eof() {
2401 let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"1,a\n"))]);
2402 let mut output = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange::new(0, 7), false, 0, 8);
2403
2404 assert_eq!(output.next().await.expect("first stream item").expect("first record"), b"1,a\n"[..]);
2405 let err = output
2406 .next()
2407 .await
2408 .expect("early EOF error")
2409 .expect_err("short ScanRange stream must fail");
2410 let object_store::Error::Generic { source, .. } = err else {
2411 panic!("expected generic object store error");
2412 };
2413 let source = source.downcast_ref::<std::io::Error>().expect("I/O error source");
2414 assert_eq!(source.kind(), std::io::ErrorKind::UnexpectedEof);
2415 assert!(source.to_string().contains("4 bytes remaining"));
2416 assert!(output.next().await.is_none());
2417 }
2418
2419 #[test]
2420 fn test_find_delimiter_handles_multi_byte_delimiter() {
2421 assert_eq!(find_delimiter(b"one\ntwo", b"\n"), Some(3));
2422 assert_eq!(find_delimiter(b"one\r\ntwo", b"\r\n"), Some(3));
2423 assert_eq!(find_delimiter(b"one\ntwo", b"\r\n"), None);
2424 assert_eq!(find_delimiter(b"one\ntwo", b""), None);
2425 }
2426
2427 #[test]
2428 fn test_scan_range_end_only_uses_aws_suffix_semantics() {
2429 let range = scan_range_from_bounds(None, Some(35), 100).unwrap().unwrap();
2430 assert_eq!(range.start(), 65);
2431 assert_eq!(range.end(), 99);
2432 }
2433
2434 #[test]
2435 fn test_scan_range_start_after_object_is_rejected_before_reader() {
2436 let err = scan_range_from_bounds(Some(100), None, 100).unwrap_err();
2437 assert!(err.to_string().contains("ScanRange"));
2438 }
2439
2440 #[test]
2441 fn test_scan_range_start_after_end_is_rejected() {
2442 let err = scan_range_from_bounds(Some(20), Some(10), 100).unwrap_err();
2443 assert!(err.to_string().contains("ScanRange"));
2444 }
2445
2446 #[test]
2447 fn test_get_range_conversion_for_parquet_bounded_ranges() {
2448 let range = http_range_spec_from_get_range(&GetRange::Bounded(10..20));
2449 assert!(!range.is_suffix_length);
2450 assert_eq!(range.start, 10);
2451 assert_eq!(range.end, 19);
2452 }
2453
2454 #[tokio::test]
2455 async fn test_scan_range_output_can_convert_field_delimiter() {
2456 let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"a&&1\nb&&2\n"))]);
2457 let stream = scan_range_stream(chunks, b"\n".to_vec(), SelectScanRange::new(0, 10), false, 0, 10);
2458 let mut stream = convert_field_delimiter_stream(stream, "&&".to_string());
2459 let mut output = Vec::new();
2460 while let Some(bytes) = stream.next().await {
2461 output.extend_from_slice(&bytes.unwrap());
2462 }
2463 assert_eq!(output, b"a,1\nb,2\n");
2464 }
2465
2466 #[tokio::test]
2467 async fn delimiter_conversion_keeps_uncompressed_metrics_equal() {
2468 let input = Bytes::from_static(b"a&&1\nb&&2\n");
2469 let input_metrics = Arc::new(SelectInputMetrics::default());
2470 let stream = stream::iter([Ok::<_, object_store::Error>(input.clone())]);
2471 let stream = meter_uncompressed_input_stream(stream, Arc::clone(&input_metrics));
2472 let output = convert_field_delimiter_stream(stream, "&&".to_string())
2473 .try_collect::<Vec<_>>()
2474 .await
2475 .expect("delimiter conversion should succeed")
2476 .concat();
2477
2478 assert_eq!(output, b"a,1\nb,2\n");
2479 let input_len = u64::try_from(input.len()).expect("fixture length should fit in u64");
2480 assert_eq!(input_metrics.snapshot().bytes_scanned, input_len);
2481 assert_eq!(input_metrics.snapshot().bytes_processed, input_len);
2482 }
2483
2484 #[tokio::test]
2485 async fn test_field_delimiter_stream_converts_delimiter_split_across_chunks() {
2486 let chunks = stream::iter(vec![
2487 Ok::<_, object_store::Error>(Bytes::from_static(b"a&")),
2488 Ok::<_, object_store::Error>(Bytes::from_static(b"&1\nb&&2\n")),
2489 ]);
2490 let mut stream = convert_field_delimiter_stream(chunks, "&&".to_string());
2491 let mut output = Vec::new();
2492 while let Some(bytes) = stream.next().await {
2493 output.extend_from_slice(&bytes.unwrap());
2494 }
2495 assert_eq!(output, b"a,1\nb,2\n");
2496 }
2497
2498 #[tokio::test]
2499 async fn test_field_delimiter_stream_converts_delimiter_at_stream_end() {
2500 let chunks = stream::iter(vec![
2501 Ok::<_, object_store::Error>(Bytes::from_static(b"a&")),
2502 Ok::<_, object_store::Error>(Bytes::from_static(b"&")),
2503 ]);
2504 let mut stream = convert_field_delimiter_stream(chunks, "&&".to_string());
2505 let mut output = Vec::new();
2506 while let Some(bytes) = stream.next().await {
2507 output.extend_from_slice(&bytes.unwrap());
2508 }
2509 assert_eq!(output, b"a,");
2510 }
2511
2512 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2513 #[serial_test::serial]
2514 async fn test_scan_range_self_overlapping_delimiter_retains_record_context() {
2515 let env = crate::storage_api::select_test_ecstore_env().await;
2516 let bucket = "s3select-scan-range-record-context";
2517 let object = "input.csv";
2518 env.make_bucket(bucket, false).await;
2519 let mut reader = SelectPutObjReader::from_vec(b"111aaa222aa333aa".to_vec());
2520 env.ecstore
2521 .put_object(bucket, object, &mut reader, &Default::default())
2522 .await
2523 .expect("put self-overlapping delimiter ScanRange fixture");
2524
2525 let make_store = |start, end, file_header_info, snapshot| EcObjectStore {
2526 input: Arc::new(SelectObjectContentInput {
2527 bucket: bucket.to_string(),
2528 expected_bucket_owner: None,
2529 key: object.to_string(),
2530 sse_customer_algorithm: None,
2531 sse_customer_key: None,
2532 sse_customer_key_md5: None,
2533 request: SelectObjectContentRequest {
2534 expression: "SELECT * FROM s3object".to_string(),
2535 expression_type: ExpressionType::from_static(ExpressionType::SQL),
2536 input_serialization: InputSerialization {
2537 csv: Some(CSVInput {
2538 record_delimiter: Some("aa".to_string()),
2539 file_header_info,
2540 ..Default::default()
2541 }),
2542 ..Default::default()
2543 },
2544 output_serialization: OutputSerialization {
2545 csv: Some(CSVOutput::default()),
2546 ..Default::default()
2547 },
2548 request_progress: None,
2549 scan_range: Some(ScanRange {
2550 start: Some(start),
2551 end: Some(end),
2552 }),
2553 },
2554 }),
2555 need_convert: false,
2556 delimiter: String::new(),
2557 is_json_document: false,
2558 json_source: JsonSource::default(),
2559 input_metrics: Arc::new(SelectInputMetrics::default()),
2560 memory_pool: Arc::new(GreedyMemoryPool::new(1024)),
2561 query_tracker: None,
2562 store: None,
2563 snapshot: OnceCell::new_with(Some(snapshot)),
2564 reader_open_count: Arc::new(AtomicUsize::new(0)),
2565 };
2566
2567 let store = make_store(6, 6, None, prepare_test_snapshot(bucket, object).await);
2568 let result = store
2569 .get_opts(&Path::from(object), GetOptions::default())
2570 .await
2571 .expect("read ScanRange starting inside record data");
2572 let GetResultPayload::Stream(stream) = result.payload else {
2573 panic!("expected streaming ScanRange payload");
2574 };
2575 let chunks: Vec<Bytes> = stream.try_collect().await.expect("collect record-data ScanRange output");
2576 assert!(chunks.concat().is_empty());
2577 drop(store);
2578
2579 let store = make_store(4, 5, None, prepare_test_snapshot(bucket, object).await);
2580 let result = store
2581 .get_opts(&Path::from(object), GetOptions::default())
2582 .await
2583 .expect("read ScanRange starting inside overlapping delimiter");
2584 let GetResultPayload::Stream(stream) = result.payload else {
2585 panic!("expected streaming overlapping-delimiter payload");
2586 };
2587 let chunks: Vec<Bytes> = stream
2588 .try_collect()
2589 .await
2590 .expect("collect overlapping-delimiter ScanRange output");
2591 assert_eq!(chunks.concat(), b"a222\r\n");
2592 drop(store);
2593
2594 let mut reader = SelectPutObjReader::from_vec(b"111aa222aa333aa".to_vec());
2595 env.ecstore
2596 .put_object(bucket, object, &mut reader, &Default::default())
2597 .await
2598 .expect("put exact review ScanRange fixture");
2599 let result = make_store(6, 6, None, prepare_test_snapshot(bucket, object).await)
2600 .get_opts(&Path::from(object), GetOptions::default())
2601 .await
2602 .expect("read exact review ScanRange fixture");
2603 let GetResultPayload::Stream(stream) = result.payload else {
2604 panic!("expected streaming exact review ScanRange payload");
2605 };
2606 let chunks: Vec<Bytes> = stream.try_collect().await.expect("collect exact review ScanRange output");
2607 assert!(chunks.concat().is_empty());
2608
2609 let result = make_store(5, 5, None, prepare_test_snapshot(bucket, object).await)
2610 .get_opts(&Path::from(object), GetOptions::default())
2611 .await
2612 .expect("read ScanRange starting after an even delimiter run");
2613 let GetResultPayload::Stream(stream) = result.payload else {
2614 panic!("expected streaming even-run ScanRange payload");
2615 };
2616 let chunks: Vec<Bytes> = stream.try_collect().await.expect("collect even-run ScanRange output");
2617 assert_eq!(chunks.concat(), b"222\r\n");
2618
2619 let mut reader = SelectPutObjReader::from_vec(b"h1aav1aav2aa".to_vec());
2620 env.ecstore
2621 .put_object(bucket, object, &mut reader, &Default::default())
2622 .await
2623 .expect("put ScanRange header snapshot fixture");
2624 let result = make_store(
2625 8,
2626 8,
2627 Some(FileHeaderInfo::from_static(FileHeaderInfo::USE)),
2628 prepare_test_snapshot(bucket, object).await,
2629 )
2630 .get_opts(&Path::from(object), GetOptions::default())
2631 .await
2632 .expect("read ScanRange with a separate header read");
2633 let GetResultPayload::Stream(stream) = result.payload else {
2634 panic!("expected streaming ScanRange header payload");
2635 };
2636 let chunks: Vec<Bytes> = stream.try_collect().await.expect("collect ScanRange header output");
2637 assert_eq!(chunks.concat(), b"h1\r\nv2\r\n");
2638
2639 let mut reader = SelectPutObjReader::from_vec(b"aaa222aa".to_vec());
2640 env.ecstore
2641 .put_object(bucket, object, &mut reader, &Default::default())
2642 .await
2643 .expect("put object-start delimiter context fixture");
2644 let result = make_store(3, 3, None, prepare_test_snapshot(bucket, object).await)
2645 .get_opts(&Path::from(object), GetOptions::default())
2646 .await
2647 .expect("read delimiter context that reaches the object start");
2648 let GetResultPayload::Stream(stream) = result.payload else {
2649 panic!("expected streaming object-start context payload");
2650 };
2651 let chunks: Vec<Bytes> = stream.try_collect().await.expect("collect object-start context output");
2652 assert!(chunks.concat().is_empty());
2653
2654 let run_start = SELECT_DEFAULT_READ_BUFFER_SIZE + 7;
2655 let mut large_fixture = vec![b'b'; run_start];
2656 large_fixture.extend_from_slice(b"aaa222aa");
2657 let mut reader = SelectPutObjReader::from_vec(large_fixture);
2658 env.ecstore
2659 .put_object(bucket, object, &mut reader, &Default::default())
2660 .await
2661 .expect("put large self-overlapping delimiter ScanRange fixture");
2662 let scan_start = i64::try_from(run_start + 3).expect("fixture offset should fit in i64");
2663 let result = make_store(scan_start, scan_start, None, prepare_test_snapshot(bucket, object).await)
2664 .get_opts(&Path::from(object), GetOptions::default())
2665 .await
2666 .expect("read large ScanRange with bounded delimiter context");
2667 let GetResultPayload::Stream(stream) = result.payload else {
2668 panic!("expected streaming large ScanRange payload");
2669 };
2670 let chunks: Vec<Bytes> = stream.try_collect().await.expect("collect large ScanRange output");
2671 assert!(chunks.concat().is_empty());
2672
2673 let mut oversized_run = vec![b'b'];
2674 oversized_run.resize(SELECT_DEFAULT_READ_BUFFER_SIZE + 2, b'a');
2675 let scan_start = i64::try_from(oversized_run.len()).expect("fixture offset should fit in i64");
2676 oversized_run.extend_from_slice(b"222aa");
2677 let mut reader = SelectPutObjReader::from_vec(oversized_run);
2678 env.ecstore
2679 .put_object(bucket, object, &mut reader, &Default::default())
2680 .await
2681 .expect("put oversized delimiter context fixture");
2682 let err = make_store(scan_start, scan_start, None, prepare_test_snapshot(bucket, object).await)
2683 .get_opts(&Path::from(object), GetOptions::default())
2684 .await
2685 .expect_err("oversized self-overlapping delimiter context must fail closed");
2686 assert!(err.to_string().contains("bounded ScanRange context"));
2687 }
2688
2689 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2690 #[serial_test::serial]
2691 async fn test_self_overlapping_record_delimiter_uses_single_full_file_partition() {
2692 const TARGET_PARTITIONS: usize = 4;
2693
2694 let mut input_bytes = Vec::with_capacity(SELECT_DEFAULT_READ_BUFFER_SIZE + 8);
2695 input_bytes.extend_from_slice(b"0,");
2696 input_bytes.resize(SELECT_DEFAULT_READ_BUFFER_SIZE - 1, b'b');
2697 input_bytes.extend_from_slice(b"aaaX,caa");
2698 assert!(input_bytes.len() > 1024 * 1024);
2699 assert_eq!(
2700 &input_bytes[SELECT_DEFAULT_READ_BUFFER_SIZE - 1..SELECT_DEFAULT_READ_BUFFER_SIZE + 2],
2701 b"aaa"
2702 );
2703
2704 let env = crate::storage_api::select_test_ecstore_env().await;
2705 let bucket = "s3select-record-delimiter";
2706 let object = "input.csv";
2707 env.make_bucket(bucket, false).await;
2708 let mut reader = SelectPutObjReader::from_vec(input_bytes);
2709 env.ecstore
2710 .put_object(bucket, object, &mut reader, &Default::default())
2711 .await
2712 .expect("put multi-byte record-delimited test object");
2713
2714 let input = Arc::new(SelectObjectContentInput {
2715 bucket: bucket.to_string(),
2716 expected_bucket_owner: None,
2717 key: object.to_string(),
2718 sse_customer_algorithm: None,
2719 sse_customer_key: None,
2720 sse_customer_key_md5: None,
2721 request: SelectObjectContentRequest {
2722 expression: "SELECT * FROM s3object".to_string(),
2723 expression_type: ExpressionType::from_static(ExpressionType::SQL),
2724 input_serialization: InputSerialization {
2725 csv: Some(CSVInput {
2726 record_delimiter: Some("aa".to_string()),
2727 ..Default::default()
2728 }),
2729 ..Default::default()
2730 },
2731 output_serialization: OutputSerialization {
2732 csv: Some(CSVOutput::default()),
2733 ..Default::default()
2734 },
2735 request_progress: None,
2736 scan_range: None,
2737 },
2738 });
2739 let snapshot = prepare_test_snapshot(bucket, object).await;
2740 let store = Arc::new(EcObjectStore {
2741 input,
2742 need_convert: false,
2743 delimiter: String::new(),
2744 is_json_document: false,
2745 json_source: JsonSource::default(),
2746 input_metrics: Arc::new(SelectInputMetrics::default()),
2747 memory_pool: Arc::new(GreedyMemoryPool::new(32 * 1024 * 1024)),
2748 query_tracker: None,
2749 store: None,
2750 snapshot: OnceCell::new_with(Some(snapshot)),
2751 reader_open_count: Arc::new(AtomicUsize::new(0)),
2752 });
2753
2754 let config = SessionConfig::new()
2755 .with_repartition_file_scans(false)
2756 .with_repartition_file_min_size(0)
2757 .with_target_partitions(TARGET_PARTITIONS);
2758 let context = SessionContext::new_with_config(config);
2759 let store_url = url::Url::parse(&format!("s3://{bucket}")).expect("valid object store URL");
2760 context.runtime_env().register_object_store(&store_url, store);
2761 context
2762 .register_csv("records", &format!("s3://{bucket}/{object}"), CsvReadOptions::new().has_header(false))
2763 .await
2764 .expect("register partitioned CSV");
2765
2766 let scan_plan = context
2767 .sql("SELECT * FROM records")
2768 .await
2769 .expect("plan partitioned CSV")
2770 .create_physical_plan()
2771 .await
2772 .expect("create partitioned CSV physical plan");
2773 assert_eq!(scan_plan.output_partitioning().partition_count(), 1);
2774
2775 let batches = context
2776 .sql("SELECT column_1 FROM records")
2777 .await
2778 .expect("plan exact-result query")
2779 .collect()
2780 .await
2781 .expect("query self-overlapping record-delimited CSV");
2782 let values = batches
2783 .iter()
2784 .flat_map(|batch| {
2785 let column = batch
2786 .column(0)
2787 .as_any()
2788 .downcast_ref::<datafusion::arrow::array::StringArray>()
2789 .expect("mixed first column should be Utf8");
2790 column.iter().map(|value| value.map(str::to_string)).collect::<Vec<_>>()
2791 })
2792 .collect::<Vec<_>>();
2793 assert_eq!(values, vec![Some("0".to_string()), Some("aX".to_string())]);
2794 }
2795
2796 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2797 #[serial_test::serial]
2798 async fn test_get_opts_validates_raw_length_before_delimiter_conversion() {
2799 let env = crate::storage_api::select_test_ecstore_env().await;
2800 let bucket = "s3select-multi-byte-delimiter";
2801 let object = "input.csv";
2802 let input_bytes = b"a\r\n1^Y";
2803 env.make_bucket(bucket, false).await;
2804 let mut reader = SelectPutObjReader::from_vec(input_bytes.to_vec());
2805 env.ecstore
2806 .put_object(bucket, object, &mut reader, &Default::default())
2807 .await
2808 .expect("put multi-byte-delimited test object");
2809
2810 let input = Arc::new(SelectObjectContentInput {
2811 bucket: bucket.to_string(),
2812 expected_bucket_owner: None,
2813 key: object.to_string(),
2814 sse_customer_algorithm: None,
2815 sse_customer_key: None,
2816 sse_customer_key_md5: None,
2817 request: SelectObjectContentRequest {
2818 expression: "SELECT * FROM s3object".to_string(),
2819 expression_type: ExpressionType::from_static(ExpressionType::SQL),
2820 input_serialization: InputSerialization {
2821 csv: Some(CSVInput {
2822 field_delimiter: Some("\r\n".to_string()),
2823 record_delimiter: Some("^Y".to_string()),
2824 ..Default::default()
2825 }),
2826 ..Default::default()
2827 },
2828 output_serialization: OutputSerialization {
2829 csv: Some(CSVOutput::default()),
2830 ..Default::default()
2831 },
2832 request_progress: None,
2833 scan_range: None,
2834 },
2835 });
2836 let snapshot = prepare_test_snapshot(bucket, object).await;
2837 let input_metrics = Arc::new(SelectInputMetrics::default());
2838 let store = super::EcObjectStore {
2839 input,
2840 need_convert: true,
2841 delimiter: "\r\n".to_string(),
2842 is_json_document: false,
2843 json_source: JsonSource::default(),
2844 input_metrics: Arc::clone(&input_metrics),
2845 memory_pool: Arc::new(GreedyMemoryPool::new(1024)),
2846 query_tracker: None,
2847 store: None,
2848 snapshot: OnceCell::new_with(Some(snapshot)),
2849 reader_open_count: Arc::new(AtomicUsize::new(0)),
2850 };
2851
2852 let result = store
2853 .get_opts(&Path::from(object), GetOptions::default())
2854 .await
2855 .expect("read multi-byte-delimited test object");
2856 let GetResultPayload::Stream(stream) = result.payload else {
2857 panic!("expected streaming object payload");
2858 };
2859 let chunks: Vec<Bytes> = stream.try_collect().await.expect("collect converted object stream");
2860
2861 assert_eq!(chunks.concat(), b"a,1\r\n");
2862 let input_len = u64::try_from(input_bytes.len()).expect("fixture length should fit in u64");
2863 assert_eq!(input_metrics.snapshot().bytes_scanned, input_len);
2864 assert_eq!(input_metrics.snapshot().bytes_processed, input_len);
2865
2866 let requested_range = 3..10;
2867 let ranges = store
2868 .get_ranges(&Path::from(object), std::slice::from_ref(&requested_range))
2869 .await
2870 .expect("bounded range past EOF should return the object remainder");
2871 assert_eq!(ranges, vec![Bytes::from_static(b"1^Y")]);
2872 }
2873
2874 #[tokio::test]
2875 async fn test_bytes_stream_stops_at_content_length() {
2876 let poll_count = Arc::new(AtomicUsize::new(0));
2877 let stream_poll_count = Arc::clone(&poll_count);
2878 let source = stream::unfold(0, move |index| {
2879 let stream_poll_count = Arc::clone(&stream_poll_count);
2880 async move {
2881 stream_poll_count.fetch_add(1, Ordering::SeqCst);
2882 let bytes = match index {
2883 0 => Bytes::from_static(b"abcd"),
2884 1 => Bytes::from_static(b"efgh"),
2885 _ => return None,
2886 };
2887 Some((Ok::<_, std::io::Error>(bytes), index + 1))
2888 }
2889 });
2890
2891 let chunks: Vec<Bytes> = bytes_stream(source, 4).try_collect().await.unwrap();
2892
2893 assert_eq!(chunks, vec![Bytes::from_static(b"abcd")]);
2894 assert_eq!(poll_count.load(Ordering::SeqCst), 1);
2895 }
2896
2897 #[tokio::test]
2898 async fn metered_stream_counts_only_polled_chunks() {
2899 let poll_count = Arc::new(AtomicUsize::new(0));
2900 let source_poll_count = Arc::clone(&poll_count);
2901 let source = stream::unfold(0, move |index| {
2902 let source_poll_count = Arc::clone(&source_poll_count);
2903 async move {
2904 source_poll_count.fetch_add(1, Ordering::SeqCst);
2905 let bytes = match index {
2906 0 => Bytes::from_static(b"abcd"),
2907 1 => Bytes::from_static(b"efgh"),
2908 _ => return None,
2909 };
2910 Some((Ok::<_, std::io::Error>(bytes), index + 1))
2911 }
2912 });
2913 let input_metrics = Arc::new(SelectInputMetrics::default());
2914 let mut metered = Box::pin(meter_uncompressed_input_stream(source, Arc::clone(&input_metrics)));
2915
2916 assert_eq!(metered.next().await.expect("first chunk").expect("valid chunk"), b"abcd"[..]);
2917 drop(metered);
2918
2919 assert_eq!(input_metrics.snapshot().bytes_scanned, 4);
2920 assert_eq!(input_metrics.snapshot().bytes_processed, 4);
2921 assert_eq!(poll_count.load(Ordering::SeqCst), 1);
2922 }
2923
2924 #[tokio::test]
2925 async fn test_bytes_stream_rejects_early_eof() {
2926 let source = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"ab"))]);
2927 let output = bytes_stream(source, 4);
2928 futures::pin_mut!(output);
2929
2930 assert_eq!(output.next().await.expect("first stream item").expect("first chunk"), b"ab"[..]);
2931 let err = output
2932 .next()
2933 .await
2934 .expect("early EOF error")
2935 .expect_err("short stream must fail");
2936 let object_store::Error::Generic { store, source } = err else {
2937 panic!("expected generic object store error");
2938 };
2939 assert_eq!(store, "EcObjectStore");
2940 let source = source.downcast_ref::<std::io::Error>().expect("I/O error source");
2941 assert_eq!(source.kind(), std::io::ErrorKind::UnexpectedEof);
2942 assert!(source.to_string().contains("2 bytes remaining"));
2943 assert!(output.next().await.is_none());
2944 }
2945
2946 #[tokio::test]
2947 async fn full_and_range_object_streams_record_input_metrics() {
2948 const BUCKET: &str = "s3select-input-metrics";
2949 const OBJECT: &str = "input.csv";
2950 const DATA: &[u8] = b"id,name\n1,a\n";
2951
2952 let env = crate::storage_api::select_test_ecstore_env().await;
2953 env.make_bucket(BUCKET, false).await;
2954 let mut reader = SelectPutObjReader::from_vec(DATA.to_vec());
2955 env.ecstore
2956 .put_object(BUCKET, OBJECT, &mut reader, &Default::default())
2957 .await
2958 .expect("put input metrics fixture");
2959 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
2960 let input_metrics = Arc::new(SelectInputMetrics::default());
2961 let store = EcObjectStore::build_with_snapshot(
2962 csv_input(BUCKET, OBJECT),
2963 Arc::new(GreedyMemoryPool::new(1024 * 1024)),
2964 None,
2965 Arc::clone(&input_metrics),
2966 Arc::clone(&snapshot),
2967 JsonSource::default(),
2968 )
2969 .expect("build metrics-aware object store");
2970
2971 let result = store
2972 .get_opts(&Path::from(OBJECT), GetOptions::default())
2973 .await
2974 .expect("open full object stream");
2975 let GetResultPayload::Stream(stream) = result.payload else {
2976 panic!("expected streaming object payload");
2977 };
2978 let body = stream
2979 .try_collect::<Vec<_>>()
2980 .await
2981 .expect("read full object stream")
2982 .concat();
2983 assert_eq!(body, DATA);
2984 let data_len = u64::try_from(DATA.len()).expect("fixture length should fit in u64");
2985 assert_eq!(input_metrics.snapshot().bytes_scanned, data_len);
2986 assert_eq!(input_metrics.snapshot().bytes_processed, data_len);
2987
2988 input_metrics.reset();
2989
2990 let result = store
2991 .get_opts(
2992 &Path::from(OBJECT),
2993 GetOptions {
2994 range: Some(GetRange::Bounded(0..2)),
2995 ..Default::default()
2996 },
2997 )
2998 .await
2999 .expect("open schema-style range stream");
3000 let GetResultPayload::Stream(stream) = result.payload else {
3001 panic!("expected range stream payload");
3002 };
3003 let range = stream
3004 .try_collect::<Vec<_>>()
3005 .await
3006 .expect("read schema-style range")
3007 .concat();
3008 assert_eq!(range, b"id"[..]);
3009 assert_eq!(input_metrics.snapshot().bytes_scanned, 2);
3010 assert_eq!(input_metrics.snapshot().bytes_processed, 2);
3011 }
3012
3013 #[tokio::test]
3014 async fn unicode_csv_quotes_preserve_raw_offsets_and_metrics() {
3015 const BUCKET: &str = "s3select-unicode-csv-stream";
3016 const HEADER: &str = "عnameع,عkindع\n";
3017 const SKIP: &str = "عskipع,عzeroع\n";
3018 const ROW: &str = "عA,Bع,عAععBع\n";
3019 let data = format!("{HEADER}{SKIP}{ROW}");
3020 let env = crate::storage_api::select_test_ecstore_env().await;
3021 env.make_bucket(BUCKET, false).await;
3022 for (object, compression, range_offset) in [
3023 ("plain.csv", None, None),
3024 ("range.csv", None, Some(0)),
3025 ("range-mid-character.csv", None, Some(1)),
3026 ("gzip.csv", Some(CompressionFormat::Gzip), None),
3027 ("bzip.csv", Some(CompressionFormat::Bzip2), None),
3028 ] {
3029 let bytes = match compression {
3030 Some(format) => encode_compressed_fixture(format, data.as_bytes()).await,
3031 None => data.as_bytes().to_vec(),
3032 };
3033 let raw_size = bytes.len();
3034 let mut reader = SelectPutObjReader::from_vec(bytes);
3035 env.ecstore
3036 .put_object(BUCKET, object, &mut reader, &Default::default())
3037 .await
3038 .expect("write Unicode CSV fixture");
3039 let mut input = (*csv_input(BUCKET, object)).clone();
3040 let csv = input.request.input_serialization.csv.as_mut().expect("CSV input");
3041 csv.file_header_info = Some(FileHeaderInfo::from_static(FileHeaderInfo::USE));
3042 csv.quote_character = Some("ع".to_owned());
3043 csv.quote_escape_character = Some("\\".to_owned());
3044 csv.record_delimiter = Some("\n".to_owned());
3045 input.request.input_serialization.compression_type = compression.map(|format| {
3046 CompressionType::from_static(match format {
3047 CompressionFormat::Gzip => CompressionType::GZIP,
3048 CompressionFormat::Bzip2 => CompressionType::BZIP2,
3049 })
3050 });
3051 let start = HEADER.len() + SKIP.len();
3052 if let Some(range_offset) = range_offset {
3053 let offset = i64::try_from(start + range_offset).expect("fixture offset");
3054 input.request.scan_range = Some(ScanRange {
3055 start: Some(offset),
3056 end: Some(offset),
3057 });
3058 }
3059 let metrics = Arc::new(SelectInputMetrics::default());
3060 let store = EcObjectStore::build_with_snapshot(
3061 Arc::new(input),
3062 Arc::new(GreedyMemoryPool::new(1024 * 1024)),
3063 None,
3064 Arc::clone(&metrics),
3065 prepare_test_snapshot(BUCKET, object).await,
3066 JsonSource::default(),
3067 )
3068 .expect("snapshot store");
3069 let result = store
3070 .get_opts(&Path::from(object), GetOptions::default())
3071 .await
3072 .expect("open Unicode CSV stream");
3073 let GetResultPayload::Stream(stream) = result.payload else { panic!("CSV must remain streaming") };
3074 let output = stream.try_collect::<Vec<_>>().await.expect("normalize CSV stream").concat();
3075 let expected = match range_offset {
3076 Some(0) => "\"name\",\"kind\"\n\"A,B\",\"AعB\"\n",
3077 Some(_) => "\"name\",\"kind\"\n",
3078 None => "\"name\",\"kind\"\n\"skip\",\"zero\"\n\"A,B\",\"AعB\"\n",
3079 };
3080 assert_eq!(output, expected.as_bytes(), "object={object}");
3081 let measured = metrics.snapshot();
3082 if let Some(range_offset) = range_offset {
3083 let processed = u64::try_from(ROW.len() + 1 - range_offset + HEADER.len()).expect("raw range length");
3086 assert_eq!(measured.bytes_scanned, processed);
3087 assert_eq!(measured.bytes_processed, processed);
3088 } else {
3089 assert_eq!(measured.bytes_scanned, u64::try_from(raw_size).expect("raw length"));
3090 assert_eq!(measured.bytes_processed, u64::try_from(data.len()).expect("decoded length"));
3091 }
3092 }
3093 }
3094
3095 #[tokio::test]
3096 async fn compressed_object_uses_one_full_stream_and_rejects_internal_ranges() {
3097 const BUCKET: &str = "s3select-compressed-object";
3098 const OBJECT: &str = "input.csv";
3099 const DATA: &[u8] = b"id,name\n1,Alice\n2,Bob\n";
3100
3101 let compressed = encode_compressed_fixture(CompressionFormat::Gzip, DATA).await;
3102 let env = crate::storage_api::select_test_ecstore_env().await;
3103 env.make_bucket(BUCKET, false).await;
3104 let mut reader = SelectPutObjReader::from_vec(compressed.clone());
3105 env.ecstore
3106 .put_object(BUCKET, OBJECT, &mut reader, &Default::default())
3107 .await
3108 .expect("put compressed CSV fixture");
3109 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
3110 let mut input = (*csv_input(BUCKET, OBJECT)).clone();
3111 input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP));
3112 input.request.scan_range = Some(ScanRange {
3113 start: Some(0),
3114 end: None,
3115 });
3116 let input_metrics = Arc::new(SelectInputMetrics::default());
3117 let store = EcObjectStore::build_with_snapshot(
3118 Arc::new(input),
3119 Arc::new(GreedyMemoryPool::new(1024 * 1024)),
3120 None,
3121 Arc::clone(&input_metrics),
3122 Arc::clone(&snapshot),
3123 JsonSource::default(),
3124 )
3125 .expect("build compressed object store");
3126
3127 let error = store
3128 .get_opts(
3129 &Path::from(OBJECT),
3130 GetOptions {
3131 range: Some(GetRange::Bounded(0..1)),
3132 ..Default::default()
3133 },
3134 )
3135 .await
3136 .expect_err("compressed input must reject DataFusion byte ranges");
3137 assert_eq!(
3138 QueryError::from(DataFusionError::ObjectStore(Box::new(error))).select_error(),
3139 SelectError::UnsupportedScanRangeInput
3140 );
3141 assert_eq!(store.reader_open_count.load(Ordering::SeqCst), 0);
3142
3143 let mut scan_input = (*csv_input(BUCKET, OBJECT)).clone();
3144 scan_input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP));
3145 scan_input.request.scan_range = Some(ScanRange {
3146 start: Some(1),
3147 end: None,
3148 });
3149 let scan_store = EcObjectStore::build_with_snapshot(
3150 Arc::new(scan_input),
3151 Arc::new(GreedyMemoryPool::new(1024 * 1024)),
3152 None,
3153 Arc::new(SelectInputMetrics::default()),
3154 snapshot,
3155 JsonSource::default(),
3156 )
3157 .expect("build compressed ScanRange object store");
3158 let error = scan_store
3159 .get_opts(&Path::from(OBJECT), GetOptions::default())
3160 .await
3161 .expect_err("compressed request ScanRange must fail before object I/O");
3162 assert_eq!(
3163 QueryError::from(DataFusionError::ObjectStore(Box::new(error))).select_error(),
3164 SelectError::UnsupportedScanRangeInput
3165 );
3166 assert_eq!(scan_store.reader_open_count.load(Ordering::SeqCst), 0);
3167
3168 let result = store
3169 .get_opts(&Path::from(OBJECT), GetOptions::default())
3170 .await
3171 .expect("open full compressed object stream");
3172 let GetResultPayload::Stream(stream) = result.payload else {
3173 panic!("expected compressed stream payload");
3174 };
3175 let body = stream
3176 .try_collect::<Vec<_>>()
3177 .await
3178 .expect("decode compressed object")
3179 .concat();
3180
3181 assert_eq!(body, DATA);
3182 assert_eq!(store.reader_open_count.load(Ordering::SeqCst), 1);
3183 assert_eq!(
3184 input_metrics.snapshot().bytes_scanned,
3185 u64::try_from(compressed.len()).expect("compressed fixture length should fit in u64")
3186 );
3187 assert_eq!(
3188 input_metrics.snapshot().bytes_processed,
3189 u64::try_from(DATA.len()).expect("input fixture length should fit in u64")
3190 );
3191 }
3192
3193 #[tokio::test]
3194 async fn compressed_stream_throughput_is_independent_of_query_memory_pool() {
3195 const BUCKET: &str = "s3select-compressed-throughput";
3196 const OBJECT: &str = "input.csv.gz";
3197
3198 let data = b"a\n".repeat(1024 * 1024);
3199 let compressed = encode_compressed_fixture(CompressionFormat::Gzip, &data).await;
3200 let env = crate::storage_api::select_test_ecstore_env().await;
3201 env.make_bucket(BUCKET, false).await;
3202 let mut reader = SelectPutObjReader::from_vec(compressed);
3203 env.ecstore
3204 .put_object(BUCKET, OBJECT, &mut reader, &Default::default())
3205 .await
3206 .expect("put compressed throughput fixture");
3207 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
3208 let mut input = (*csv_input(BUCKET, OBJECT)).clone();
3209 input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP));
3210 let metrics = Arc::new(SelectInputMetrics::default());
3211 let store = EcObjectStore::build_with_snapshot(
3212 Arc::new(input),
3213 Arc::new(GreedyMemoryPool::new(1)),
3214 None,
3215 Arc::clone(&metrics),
3216 snapshot,
3217 JsonSource::default(),
3218 )
3219 .expect("build compressed throughput store");
3220
3221 let result = store
3222 .get_opts(&Path::from(OBJECT), GetOptions::default())
3223 .await
3224 .expect("open compressed throughput stream");
3225 let GetResultPayload::Stream(stream) = result.payload else {
3226 panic!("expected compressed stream payload");
3227 };
3228 let decoded = stream
3229 .try_collect::<Vec<_>>()
3230 .await
3231 .expect("streamed decoded bytes should not consume the query memory pool")
3232 .concat();
3233
3234 assert_eq!(decoded, data);
3235 assert_eq!(
3236 metrics.snapshot().bytes_processed,
3237 u64::try_from(data.len()).expect("fixture length should fit in u64")
3238 );
3239 }
3240
3241 #[tokio::test]
3242 async fn dropping_real_object_stream_counts_only_consumed_bytes() {
3243 const BUCKET: &str = "s3select-partial-input-metrics";
3244 const OBJECT: &str = "large.csv";
3245
3246 let data = vec![b'x'; SELECT_DEFAULT_READ_BUFFER_SIZE * 3];
3247 let env = crate::storage_api::select_test_ecstore_env().await;
3248 env.make_bucket(BUCKET, false).await;
3249 let mut reader = SelectPutObjReader::from_vec(data.clone());
3250 env.ecstore
3251 .put_object(BUCKET, OBJECT, &mut reader, &Default::default())
3252 .await
3253 .expect("put partial input metrics fixture");
3254 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
3255 let input_metrics = Arc::new(SelectInputMetrics::default());
3256 let store = EcObjectStore::build_with_snapshot(
3257 csv_input(BUCKET, OBJECT),
3258 Arc::new(GreedyMemoryPool::new(1024 * 1024)),
3259 None,
3260 Arc::clone(&input_metrics),
3261 snapshot,
3262 JsonSource::default(),
3263 )
3264 .expect("build metrics-aware object store");
3265
3266 let result = store
3267 .get_opts(&Path::from(OBJECT), GetOptions::default())
3268 .await
3269 .expect("open partial object stream");
3270 let GetResultPayload::Stream(mut stream) = result.payload else {
3271 panic!("expected streaming object payload");
3272 };
3273 let first = stream
3274 .next()
3275 .await
3276 .expect("first object chunk")
3277 .expect("first object chunk should be valid");
3278 drop(stream);
3279
3280 assert!(first.len() < data.len(), "fixture must span multiple reader chunks");
3281 let consumed = u64::try_from(first.len()).expect("chunk length should fit in u64");
3282 assert_eq!(input_metrics.snapshot().bytes_scanned, consumed);
3283 assert_eq!(input_metrics.snapshot().bytes_processed, consumed);
3284 }
3285
3286 #[tokio::test]
3287 async fn json_object_streams_record_input_metrics() {
3288 const BUCKET: &str = "s3select-json-input-metrics";
3289 const LINES_OBJECT: &str = "input.jsonl";
3290 const LINES_DATA: &[u8] = b"{\"id\":1}\n{\"id\":2}\n";
3291 const DOCUMENT_OBJECT: &str = "input.json";
3292 const DOCUMENT_DATA: &[u8] = b"[{\"id\":1},{\"id\":2}]";
3293
3294 let env = crate::storage_api::select_test_ecstore_env().await;
3295 env.make_bucket(BUCKET, false).await;
3296 for (object, data, json_type) in [
3297 (LINES_OBJECT, LINES_DATA, JSONType::LINES),
3298 (DOCUMENT_OBJECT, DOCUMENT_DATA, JSONType::DOCUMENT),
3299 ] {
3300 let mut reader = SelectPutObjReader::from_vec(data.to_vec());
3301 env.ecstore
3302 .put_object(BUCKET, object, &mut reader, &Default::default())
3303 .await
3304 .expect("put JSON input metrics fixture");
3305 let snapshot = prepare_test_snapshot(BUCKET, object).await;
3306 let input_metrics = Arc::new(SelectInputMetrics::default());
3307 let memory_limit = if json_type == JSONType::DOCUMENT {
3308 JSON_DOCUMENT_TEST_MEMORY_BYTES
3309 } else {
3310 1024 * 1024
3311 };
3312 let store = EcObjectStore::build_with_snapshot(
3313 json_input(BUCKET, object, json_type),
3314 Arc::new(GreedyMemoryPool::new(memory_limit)),
3315 None,
3316 Arc::clone(&input_metrics),
3317 snapshot,
3318 JsonSource::default(),
3319 )
3320 .expect("build metrics-aware JSON object store");
3321
3322 let result = store
3323 .get_opts(&Path::from(object), GetOptions::default())
3324 .await
3325 .expect("open JSON object stream");
3326 let GetResultPayload::Stream(stream) = result.payload else {
3327 panic!("expected streaming JSON payload");
3328 };
3329 stream.try_collect::<Vec<_>>().await.expect("read JSON object stream");
3330
3331 let input_len = u64::try_from(data.len()).expect("fixture length should fit in u64");
3332 assert_eq!(input_metrics.snapshot().bytes_scanned, input_len, "JSON type {json_type}");
3333 assert_eq!(input_metrics.snapshot().bytes_processed, input_len, "JSON type {json_type}");
3334 }
3335 }
3336
3337 #[tokio::test]
3338 async fn scan_range_metrics_include_header_and_raw_range_once() {
3339 const BUCKET: &str = "s3select-scan-range-input-metrics";
3340 const OBJECT: &str = "input.csv";
3341 const DATA: &[u8] = b"h1,h2\nr1,a\nr2,b\n";
3342 const HEADER_LEN: usize = b"h1,h2\n".len();
3343 const RECORD_START: usize = b"h1,h2\nr1,a\n".len();
3344 const READ_START: usize = RECORD_START - 1;
3345
3346 let env = crate::storage_api::select_test_ecstore_env().await;
3347 env.make_bucket(BUCKET, false).await;
3348 let mut reader = SelectPutObjReader::from_vec(DATA.to_vec());
3349 env.ecstore
3350 .put_object(BUCKET, OBJECT, &mut reader, &Default::default())
3351 .await
3352 .expect("put ScanRange input metrics fixture");
3353 let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
3354 let mut input = (*csv_input(BUCKET, OBJECT)).clone();
3355 input
3356 .request
3357 .input_serialization
3358 .csv
3359 .as_mut()
3360 .expect("CSV input")
3361 .file_header_info = Some(FileHeaderInfo::from_static(FileHeaderInfo::USE));
3362 input.request.scan_range = Some(ScanRange {
3363 start: Some(i64::try_from(RECORD_START).expect("fixture offset should fit in i64")),
3364 end: Some(i64::try_from(RECORD_START).expect("fixture offset should fit in i64")),
3365 });
3366 let input_metrics = Arc::new(SelectInputMetrics::default());
3367 let store = EcObjectStore::build_with_snapshot(
3368 Arc::new(input),
3369 Arc::new(GreedyMemoryPool::new(1024 * 1024)),
3370 None,
3371 Arc::clone(&input_metrics),
3372 snapshot,
3373 JsonSource::default(),
3374 )
3375 .expect("build ScanRange metrics-aware object store");
3376
3377 let result = store
3378 .get_opts(&Path::from(OBJECT), GetOptions::default())
3379 .await
3380 .expect("open ScanRange object stream");
3381 let GetResultPayload::Stream(stream) = result.payload else {
3382 panic!("expected streaming ScanRange payload");
3383 };
3384 let body = stream
3385 .try_collect::<Vec<_>>()
3386 .await
3387 .expect("read ScanRange object stream")
3388 .concat();
3389
3390 assert_eq!(body, b"h1,h2\nr2,b\n");
3391 let expected_input = u64::try_from(HEADER_LEN + DATA.len() - READ_START).expect("fixture length should fit in u64");
3392 assert_eq!(input_metrics.snapshot().bytes_scanned, expected_input);
3393 assert_eq!(input_metrics.snapshot().bytes_processed, expected_input);
3394 }
3395
3396 #[tokio::test]
3397 async fn test_json_document_stream_respects_query_memory_pool() {
3398 let input = b"{}".to_vec();
3399 let memory_pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(1));
3400 let mut output = json_document_ndjson_stream(
3401 Box::new(std::io::Cursor::new(input.clone())),
3402 input.len() as u64,
3403 JsonSource::default(),
3404 Arc::new(SelectInputMetrics::default()),
3405 memory_pool,
3406 None,
3407 )
3408 .expect("build constrained JSON document stream");
3409
3410 let err = output
3411 .next()
3412 .await
3413 .expect("memory error")
3414 .expect_err("reservation should exceed the pool");
3415 let object_store::Error::Generic { source, .. } = err else {
3416 panic!("expected generic object store error");
3417 };
3418 assert!(matches!(
3419 source.downcast_ref::<DataFusionError>(),
3420 Some(DataFusionError::ResourcesExhausted(_))
3421 ));
3422 }
3423
3424 #[tokio::test]
3425 async fn compressed_json_document_uses_decoded_size_and_metric_boundaries() {
3426 const INPUT: &[u8] = br#"[{"id":1},{"id":2}]"#;
3427 const EXPECTED: &[u8] = b"{\"id\":1}\n{\"id\":2}\n";
3428
3429 for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] {
3430 let compressed = encode_compressed_fixture(format, INPUT).await;
3431 let compressed_len = u64::try_from(compressed.len()).expect("compressed fixture length should fit in u64");
3432 let input_metrics = Arc::new(SelectInputMetrics::default());
3433 let reader = compressed_input_reader(
3434 Box::new(std::io::Cursor::new(compressed)),
3435 compressed_len,
3436 format,
3437 Arc::clone(&input_metrics),
3438 u64::MAX,
3439 None,
3440 );
3441 let output = compressed_json_document_ndjson_stream(
3442 reader,
3443 JsonSource::default(),
3444 Arc::new(GreedyMemoryPool::new(JSON_DOCUMENT_TEST_MEMORY_BYTES)),
3445 None,
3446 )
3447 .try_collect::<Vec<_>>()
3448 .await
3449 .expect("compressed JSON DOCUMENT should decode and parse")
3450 .concat();
3451
3452 assert_eq!(output, EXPECTED);
3453 assert_eq!(input_metrics.snapshot().bytes_scanned, compressed_len);
3454 assert_eq!(
3455 input_metrics.snapshot().bytes_processed,
3456 u64::try_from(INPUT.len()).expect("JSON fixture length should fit in u64")
3457 );
3458 }
3459 }
3460
3461 async fn compressed_json_document_select_error(
3462 format: CompressionFormat,
3463 compressed: Vec<u8>,
3464 max_processed_bytes: u64,
3465 ) -> SelectError {
3466 let compressed_len = u64::try_from(compressed.len()).expect("compressed fixture length should fit in u64");
3467 let reader = compressed_input_reader(
3468 Box::new(std::io::Cursor::new(compressed)),
3469 compressed_len,
3470 format,
3471 Arc::new(SelectInputMetrics::default()),
3472 max_processed_bytes,
3473 None,
3474 );
3475 let mut output = compressed_json_document_ndjson_stream(
3476 reader,
3477 JsonSource::default(),
3478 Arc::new(GreedyMemoryPool::new(64 * 1024 * 1024)),
3479 None,
3480 );
3481 while let Some(result) = output.next().await {
3482 if let Err(source) = result {
3483 assert!(output.next().await.is_none());
3484 return QueryError::from(DataFusionError::ObjectStore(Box::new(source))).select_error();
3485 }
3486 }
3487 panic!("compressed JSON DOCUMENT decoding must fail");
3488 }
3489
3490 #[tokio::test]
3491 async fn compressed_json_document_preserves_decoder_select_errors() {
3492 const INPUT: &[u8] = br#"[{"id":1}]"#;
3493
3494 for (format, compression) in [(CompressionFormat::Gzip, "GZIP"), (CompressionFormat::Bzip2, "BZIP2")] {
3495 assert_eq!(
3496 compressed_json_document_select_error(format, b"not compressed".to_vec(), u64::MAX).await,
3497 SelectError::InvalidCompressionFormatForObject { compression }
3498 );
3499
3500 let mut truncated = encode_compressed_fixture(format, INPUT).await;
3501 truncated.truncate(truncated.len() - 1);
3502 assert_eq!(
3503 compressed_json_document_select_error(format, truncated, u64::MAX).await,
3504 SelectError::TruncatedInput
3505 );
3506
3507 let compressed = encode_compressed_fixture(format, INPUT).await;
3508 let max_processed_bytes = u64::try_from(INPUT.len() - 1).expect("fixture length should fit in u64");
3509 assert_eq!(
3510 compressed_json_document_select_error(format, compressed, max_processed_bytes).await,
3511 SelectError::ResourceExhausted
3512 );
3513 }
3514 }
3515
3516 #[tokio::test]
3517 async fn json_document_input_record_limit_applies_after_decompression() {
3518 let at_limit = escaped_json_object_with_size(MAX_SELECT_RECORD_BYTES);
3519 let over_limit = escaped_json_object_with_size(MAX_SELECT_RECORD_BYTES + 1);
3520
3521 for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] {
3522 let compressed = encode_compressed_fixture(format, &at_limit).await;
3523 let compressed_len = u64::try_from(compressed.len()).expect("compressed fixture length should fit in u64");
3524 let reader = compressed_input_reader(
3525 Box::new(std::io::Cursor::new(compressed)),
3526 compressed_len,
3527 format,
3528 Arc::new(SelectInputMetrics::default()),
3529 u64::MAX,
3530 None,
3531 );
3532 compressed_json_document_ndjson_stream(
3533 reader,
3534 JsonSource::default(),
3535 Arc::new(GreedyMemoryPool::new(64 * 1024 * 1024)),
3536 None,
3537 )
3538 .try_collect::<Vec<_>>()
3539 .await
3540 .expect("one-megabyte compressed input record should be accepted");
3541
3542 let compressed = encode_compressed_fixture(format, &over_limit).await;
3543 assert_eq!(
3544 compressed_json_document_select_error(format, compressed, u64::MAX).await,
3545 SelectError::OverMaxRecordSize
3546 );
3547 }
3548 }
3549
3550 #[tokio::test]
3551 async fn test_json_document_stream_releases_memory_reservation() {
3552 let input = b"[1,2]".to_vec();
3553 let memory_pool = Arc::new(GreedyMemoryPool::new(JSON_DOCUMENT_TEST_MEMORY_BYTES));
3554 let input_metrics = Arc::new(SelectInputMetrics::default());
3555 let output: Vec<Bytes> = json_document_ndjson_stream(
3556 Box::new(std::io::Cursor::new(input.clone())),
3557 input.len() as u64,
3558 JsonSource::default(),
3559 Arc::clone(&input_metrics),
3560 memory_pool.clone(),
3561 None,
3562 )
3563 .expect("build JSON document stream")
3564 .try_collect()
3565 .await
3566 .expect("JSON conversion should fit the pool");
3567 let input_len = u64::try_from(input.len()).expect("fixture length should fit in u64");
3568 assert_eq!(input_metrics.snapshot().bytes_scanned, input_len);
3569 assert_eq!(input_metrics.snapshot().bytes_processed, input_len);
3570
3571 assert_eq!(output, vec![Bytes::from_static(b"{\"_1\":1}\n"), Bytes::from_static(b"{\"_1\":2}\n")]);
3572 assert_eq!(memory_pool.reserved(), 0);
3573 }
3574
3575 #[tokio::test]
3576 async fn test_json_document_stream_reports_early_eof_after_completed_rows() {
3577 let input = b"{}".to_vec();
3578 let input_len = u64::try_from(input.len()).expect("fixture length should fit in u64");
3579 let input_metrics = Arc::new(SelectInputMetrics::default());
3580 let memory_pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(JSON_DOCUMENT_TEST_MEMORY_BYTES));
3581 let mut output = json_document_ndjson_stream(
3582 Box::new(std::io::Cursor::new(input)),
3583 4,
3584 JsonSource::default(),
3585 Arc::clone(&input_metrics),
3586 memory_pool,
3587 None,
3588 )
3589 .expect("build short JSON document stream");
3590
3591 let row = output
3592 .next()
3593 .await
3594 .expect("completed row")
3595 .expect("completed rows may precede a terminal stream error");
3596 assert_eq!(row, Bytes::from_static(b"{}\n"));
3597
3598 let err = output
3599 .next()
3600 .await
3601 .expect("early EOF error")
3602 .expect_err("short JSON document must fail");
3603 let object_store::Error::Generic { source, .. } = err else {
3604 panic!("expected generic object store error");
3605 };
3606 let source = source.downcast_ref::<std::io::Error>().expect("I/O error source");
3607 assert_eq!(source.kind(), std::io::ErrorKind::UnexpectedEof);
3608 assert!(source.to_string().contains("2 bytes remaining"));
3609 assert_eq!(input_metrics.snapshot().bytes_scanned, input_len);
3610 assert_eq!(input_metrics.snapshot().bytes_processed, input_len);
3611 assert!(output.next().await.is_none());
3612 }
3613
3614 #[tokio::test]
3615 async fn malformed_json_document_stream_has_typed_select_error() {
3616 let input = b"{bad".to_vec();
3617 let memory_pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(JSON_DOCUMENT_TEST_MEMORY_BYTES));
3618 let mut output = json_document_ndjson_stream(
3619 Box::new(std::io::Cursor::new(input.clone())),
3620 input.len() as u64,
3621 JsonSource::default(),
3622 Arc::new(SelectInputMetrics::default()),
3623 memory_pool,
3624 None,
3625 )
3626 .expect("build malformed JSON document stream");
3627
3628 let source = output
3629 .next()
3630 .await
3631 .expect("malformed JSON should produce one stream error")
3632 .expect_err("malformed JSON DOCUMENT must fail");
3633 let error = QueryError::from(DataFusionError::ObjectStore(Box::new(source)));
3634
3635 assert_eq!(error.select_error(), SelectError::JsonParsingError);
3636 assert!(output.next().await.is_none());
3637 }
3638
3639 #[test]
3640 fn storage_error_mapper_preserves_protocol_classification() {
3641 let classify = |source| QueryError::from(DataFusionError::ObjectStore(Box::new(source))).select_error();
3642
3643 assert_eq!(
3644 classify(map_storage_error(
3645 "private-bucket",
3646 "private-object",
3647 SelectStorageError::BucketNotFound("private-bucket".to_string()),
3648 )),
3649 SelectError::BucketNotFound
3650 );
3651 assert_eq!(
3652 classify(map_storage_error(
3653 "private-bucket",
3654 "private-object",
3655 SelectStorageError::ObjectNotFound("private-bucket".to_string(), "private-object".to_string()),
3656 )),
3657 SelectError::ObjectNotFound
3658 );
3659 assert_eq!(
3660 classify(map_storage_error("private-bucket", "private-object", SelectStorageError::LessData)),
3661 SelectError::InternalError
3662 );
3663 assert_eq!(
3664 classify(scan_range_from_bounds(Some(10), None, 10).expect_err("out-of-bounds range must fail")),
3665 SelectError::InvalidScanRange
3666 );
3667 let parquet_source = map_storage_error(
3668 "private-bucket",
3669 "private-object",
3670 SelectStorageError::ObjectNotFound("private-bucket".to_string(), "private-object".to_string()),
3671 );
3672 let parquet_error = QueryError::from(DataFusionError::ParquetError(Box::new(
3673 datafusion::parquet::errors::ParquetError::External(Box::new(parquet_source)),
3674 )));
3675 assert_eq!(parquet_error.select_error(), SelectError::ObjectNotFound);
3676 }
3677
3678 #[test]
3679 fn test_json_document_size_error_is_resource_exhausted() {
3680 assert!(validate_json_document_size(super::MAX_JSON_DOCUMENT_BYTES).is_ok());
3681
3682 let err = validate_json_document_size(super::MAX_JSON_DOCUMENT_BYTES + 1).expect_err("oversized JSON document must fail");
3683 let object_store::Error::Generic { source, .. } = err else {
3684 panic!("expected generic object store error");
3685 };
3686 assert!(matches!(
3687 source.downcast_ref::<DataFusionError>(),
3688 Some(DataFusionError::ResourcesExhausted(_))
3689 ));
3690 }
3691
3692 #[test]
3693 fn test_json_document_parser_does_not_use_tokio_blocking_pool() {
3694 let runtime = tokio::runtime::Builder::new_multi_thread()
3695 .worker_threads(2)
3696 .max_blocking_threads(1)
3697 .enable_all()
3698 .build()
3699 .expect("build test runtime");
3700
3701 runtime.block_on(async {
3702 let (blocking_started_tx, blocking_started_rx) = tokio::sync::oneshot::channel();
3703 let (release_blocking_tx, release_blocking_rx) = std::sync::mpsc::channel();
3704 let blocker = tokio::task::spawn_blocking(move || {
3705 let _ = blocking_started_tx.send(());
3706 release_blocking_rx.recv().expect("release blocking worker");
3707 });
3708 blocking_started_rx.await.expect("blocking worker should start");
3709
3710 let admission = Arc::new(Semaphore::new(1));
3711 let permit = Arc::clone(&admission)
3712 .acquire_owned()
3713 .await
3714 .expect("query permit should be available");
3715 let query_guard: QueryExecutionGuard = Arc::new(permit);
3716 let query_tracker = QueryExecutionTracker::new(
3717 &QueryExecutionOwner::new(),
3718 query_guard,
3719 tokio::time::Instant::now() + std::time::Duration::from_secs(30),
3720 30,
3721 );
3722 let input = b"{}".to_vec();
3723 let (memory_pool, reservation_released) =
3724 AdmissionObservingMemoryPool::new(JSON_DOCUMENT_TEST_MEMORY_BYTES, Arc::clone(&admission));
3725 let memory_pool = Arc::new(memory_pool);
3726 let query_memory_pool: Arc<dyn MemoryPool> = memory_pool.clone();
3727 let mut output = json_document_ndjson_stream(
3728 Box::new(std::io::Cursor::new(input.clone())),
3729 input.len() as u64,
3730 JsonSource::default(),
3731 Arc::new(SelectInputMetrics::default()),
3732 query_memory_pool,
3733 Some(query_tracker),
3734 )
3735 .expect("build JSON document stream");
3736
3737 let row = tokio::time::timeout(std::time::Duration::from_secs(1), output.next())
3738 .await
3739 .expect("dedicated parser must not wait for Tokio's blocking pool")
3740 .expect("JSON document should produce one row")
3741 .expect("JSON document row should be valid");
3742 assert_eq!(row, Bytes::from_static(b"{}\n"));
3743 assert!(
3744 tokio::time::timeout(std::time::Duration::from_secs(1), output.next())
3745 .await
3746 .expect("parser should finish while the blocking pool is occupied")
3747 .is_none()
3748 );
3749 drop(output);
3750
3751 tokio::pin!(reservation_released);
3752 assert!(
3753 tokio::time::timeout(std::time::Duration::from_secs(1), &mut reservation_released)
3754 .await
3755 .expect("completed parser should release memory")
3756 .expect("memory reservation release observer should remain open"),
3757 "query admission must cover the memory reservation through teardown"
3758 );
3759 assert_eq!(memory_pool.reserved(), 0);
3760 let recovered_permit =
3761 tokio::time::timeout(std::time::Duration::from_secs(1), Arc::clone(&admission).acquire_owned())
3762 .await
3763 .expect("completed parser should release query admission")
3764 .expect("query admission should remain open");
3765 drop(recovered_permit);
3766
3767 release_blocking_tx.send(()).expect("release blocking worker");
3768 blocker.await.expect("blocking worker should finish");
3769 assert_eq!(admission.available_permits(), 1);
3770 });
3771 }
3772
3773 #[test]
3775 fn test_flatten_array_produces_one_line_per_element() {
3776 let input = br#"[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]"#;
3777 let result = flatten_json_document_to_ndjson(input, &[]).expect("should succeed");
3778 let text = std::str::from_utf8(&result).unwrap();
3779 let lines: Vec<&str> = text.lines().collect();
3780 assert_eq!(lines.len(), 2);
3781 for line in &lines {
3783 serde_json::from_str::<serde_json::Value>(line).expect("each line must be valid JSON");
3784 }
3785 let first: serde_json::Value = serde_json::from_str(lines[0]).unwrap();
3787 assert_eq!(first["id"], 1);
3788 assert_eq!(first["name"], "Alice");
3789 }
3790
3791 #[test]
3793 fn test_flatten_single_object_produces_one_line() {
3794 let input = br#"{"id":42,"value":"hello world"}"#;
3795 let result = flatten_json_document_to_ndjson(input, &[]).expect("should succeed");
3796 let text = std::str::from_utf8(&result).unwrap();
3797 let lines: Vec<&str> = text.lines().collect();
3798 assert_eq!(lines.len(), 1);
3799 let parsed: serde_json::Value = serde_json::from_str(lines[0]).unwrap();
3800 assert_eq!(parsed["id"], 42);
3801 assert_eq!(parsed["value"], "hello world");
3802 }
3803
3804 #[test]
3806 fn test_flatten_empty_array_produces_no_output() {
3807 let input = b"[]";
3808 let result = flatten_json_document_to_ndjson(input, &[]).expect("should succeed");
3809 assert!(result.is_empty(), "empty array should yield zero bytes");
3810 }
3811
3812 #[test]
3814 fn test_flatten_pretty_printed_document() {
3815 let input = b"[\n {\"a\": 1},\n {\"a\": 2},\n {\"a\": 3}\n]";
3816 let result = flatten_json_document_to_ndjson(input, &[]).expect("should succeed");
3817 let text = std::str::from_utf8(&result).unwrap();
3818 assert_eq!(text.lines().count(), 3);
3819 }
3820
3821 #[test]
3823 fn test_flatten_array_with_nested_objects() {
3824 let input = br#"[{"outer":{"inner":99}},{"outer":{"inner":100}}]"#;
3825 let result = flatten_json_document_to_ndjson(input, &[]).expect("should succeed");
3826 let text = std::str::from_utf8(&result).unwrap();
3827 let lines: Vec<&str> = text.lines().collect();
3828 assert_eq!(lines.len(), 2);
3829 for line in &lines {
3831 assert!(!line.is_empty());
3832 let v: serde_json::Value = serde_json::from_str(line).unwrap();
3833 assert!(v["outer"]["inner"].as_i64().unwrap() >= 99);
3834 }
3835 }
3836
3837 #[test]
3839 fn test_flatten_output_ends_with_newline_per_record() {
3840 let input = br#"[{"x":1},{"x":2}]"#;
3841 let result = flatten_json_document_to_ndjson(input, &[]).expect("should succeed");
3842 let text = std::str::from_utf8(&result).unwrap();
3843 assert_eq!(text.chars().filter(|&c| c == '\n').count(), 2);
3845 assert!(!text.starts_with('\n'));
3847 }
3848
3849 #[test]
3851 fn test_flatten_invalid_json_returns_error() {
3852 let input = b"{ not valid json }";
3853 let err = flatten_json_document_to_ndjson(input, &[]).expect_err("should fail on invalid JSON");
3854 assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
3855 }
3856
3857 #[test]
3858 fn json_document_logical_record_enforces_one_megabyte_limit() {
3859 const SCALAR_WRAPPER_BYTES: usize = br#"{"":0}"#.len();
3860 let input = br#"{"v":0}"#;
3861 let path = vec![source_key("v")];
3862
3863 let alias = "a".repeat(MAX_SELECT_RECORD_BYTES - SCALAR_WRAPPER_BYTES);
3864 let source = JsonSource::new(path.clone(), Some(alias));
3865 let line = super::parse_json_document_to_lines(input, &source).expect("one-megabyte logical record should be accepted");
3866 assert_eq!(line[0].len(), MAX_SELECT_RECORD_BYTES + 1);
3867
3868 let alias = "a".repeat(MAX_SELECT_RECORD_BYTES + 1 - SCALAR_WRAPPER_BYTES);
3869 let source = JsonSource::new(path.clone(), Some(alias));
3870 let error = super::parse_json_document_to_lines(input, &source).expect_err("oversized logical record must fail");
3871 assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
3872 assert!(error.get_ref().is_some_and(|source| {
3873 source
3874 .downcast_ref::<SelectError>()
3875 .is_some_and(|error| error == &SelectError::OverMaxRecordSize)
3876 }));
3877
3878 let escaped_alias = "\0".repeat(MAX_SELECT_RECORD_BYTES / 6);
3879 let source = JsonSource::new(path, Some(escaped_alias));
3880 let error = super::parse_json_document_to_lines(input, &source)
3881 .expect_err("escaped output must be bounded while it is serialized");
3882 assert!(error.get_ref().is_some_and(|source| {
3883 source
3884 .downcast_ref::<SelectError>()
3885 .is_some_and(|error| error == &SelectError::OverMaxRecordSize)
3886 }));
3887 }
3888
3889 #[test]
3891 fn test_flatten_empty_input_returns_error() {
3892 let err = flatten_json_document_to_ndjson(b"", &[]).expect_err("empty bytes are not valid JSON");
3893 assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
3894 }
3895
3896 #[test]
3897 fn key_path_expands_final_array_for_legacy_queries() {
3898 let input = br#"{"employees":[{"id":1,"name":"Alice","salary":75000},{"id":2,"name":"Bob","salary":65000}]}"#;
3899 let result = flatten_json_document_to_ndjson(input, &[source_key("employees")]).expect("key path should succeed");
3900 let text = std::str::from_utf8(&result).unwrap();
3901 let lines: Vec<&str> = text.lines().collect();
3902 assert_eq!(lines.len(), 2, "each employee should be its own NDJSON line");
3903 let first: serde_json::Value = serde_json::from_str(lines[0]).unwrap();
3904 assert_eq!(first["name"], "Alice");
3905 assert_eq!(first["salary"], 75000);
3906 let second: serde_json::Value = serde_json::from_str(lines[1]).unwrap();
3907 assert_eq!(second["name"], "Bob");
3908 }
3909
3910 #[test]
3911 fn nested_array_wildcards_expand_in_source_order() {
3912 let input = br#"{"departments":[{"employees":[{"id":1},{"id":2}]},{"employees":[{"id":3}]}]}"#;
3913 let path = [
3914 JsonPathSegment::ArrayWildcard,
3915 source_key("departments"),
3916 JsonPathSegment::ArrayWildcard,
3917 source_key("employees"),
3918 JsonPathSegment::ArrayWildcard,
3919 ];
3920
3921 let result = flatten_json_document_to_ndjson(input, &path).expect("nested wildcard path should succeed");
3922 let ids: Vec<i64> = std::str::from_utf8(&result)
3923 .unwrap()
3924 .lines()
3925 .map(|line| {
3926 serde_json::from_str::<serde_json::Value>(line).unwrap()["id"]
3927 .as_i64()
3928 .unwrap()
3929 })
3930 .collect();
3931
3932 assert_eq!(ids, vec![1, 2, 3]);
3933 }
3934
3935 #[test]
3936 fn explicit_array_wildcard_does_not_expand_nested_array_records_twice() {
3937 let input = br#"{"groups":[[1,2],[3,4]]}"#;
3938 let source = JsonSource::new(vec![source_key("groups"), JsonPathSegment::ArrayWildcard], Some("g".to_string()));
3939
3940 let result = super::parse_json_document_to_lines(input, &source).expect("explicit wildcard path should succeed");
3941
3942 assert_eq!(
3943 result,
3944 vec![Bytes::from_static(b"{\"g\":[1,2]}\n"), Bytes::from_static(b"{\"g\":[3,4]}\n")]
3945 );
3946 }
3947
3948 #[test]
3949 fn leading_array_wildcard_expands_a_root_array_before_nested_keys() {
3950 let input = br#"[{"employees":[{"id":1}]},{"employees":[{"id":2}]}]"#;
3951 let path = [
3952 JsonPathSegment::ArrayWildcard,
3953 source_key("employees"),
3954 JsonPathSegment::ArrayWildcard,
3955 ];
3956
3957 let result = flatten_json_document_to_ndjson(input, &path).expect("root array path should succeed");
3958 let ids: Vec<i64> = std::str::from_utf8(&result)
3959 .unwrap()
3960 .lines()
3961 .map(|line| {
3962 serde_json::from_str::<serde_json::Value>(line).unwrap()["id"]
3963 .as_i64()
3964 .unwrap()
3965 })
3966 .collect();
3967
3968 assert_eq!(ids, vec![1, 2]);
3969 }
3970
3971 #[test]
3972 fn root_array_index_selects_one_record() {
3973 let input = br#"[{"id":1},{"id":2}]"#;
3974
3975 let result = flatten_json_document_to_ndjson(input, &[JsonPathSegment::Index(1)]).expect("array index should succeed");
3976
3977 assert_eq!(result, Bytes::from_static(b"{\"id\":2}\n"));
3978 }
3979
3980 #[test]
3981 fn canonical_root_array_index_selects_one_record() {
3982 let input = br#"[{"id":1},{"id":2}]"#;
3983 let path = [JsonPathSegment::ArrayWildcard, JsonPathSegment::Index(0)];
3984
3985 let result = flatten_json_document_to_ndjson(input, &path).expect("canonical array index should succeed");
3986
3987 assert_eq!(result, Bytes::from_static(b"{\"id\":1}\n"));
3988 }
3989
3990 #[test]
3991 fn out_of_range_root_array_indexes_produce_no_records() {
3992 let input = br#"[{"id":1}]"#;
3993
3994 for path in [
3995 vec![JsonPathSegment::Index(1)],
3996 vec![JsonPathSegment::ArrayWildcard, JsonPathSegment::Index(1)],
3997 ] {
3998 let result = flatten_json_document_to_ndjson(input, &path).expect("out-of-range index should not fail");
3999 assert!(result.is_empty());
4000 }
4001 }
4002
4003 #[test]
4004 fn canonical_key_path_does_not_expand_an_array_without_a_wildcard() {
4005 let input = br#"{"rules":[{"id":1},{"id":2}]}"#;
4006 let source = JsonSource::new(vec![JsonPathSegment::ArrayWildcard, source_key("rules")], Some("r".to_string()));
4007
4008 let result = super::parse_json_document_to_lines(input, &source).expect("array-valued source path should remain one row");
4009
4010 assert_eq!(result, vec![Bytes::from_static(b"{\"r\":[{\"id\":1},{\"id\":2}]}\n")]);
4011 }
4012
4013 #[test]
4014 fn object_wildcard_expands_values_and_allows_continuation() {
4015 let input = br#"{"groups":{"first":{"id":1},"second":{"id":2}}}"#;
4016 let path = [
4017 JsonPathSegment::ArrayWildcard,
4018 source_key("groups"),
4019 JsonPathSegment::ObjectWildcard,
4020 source_key("id"),
4021 ];
4022
4023 let result = flatten_json_document_to_ndjson(input, &path).expect("object wildcard should succeed");
4024 let values: Vec<serde_json::Value> = std::str::from_utf8(&result)
4025 .unwrap()
4026 .lines()
4027 .map(|line| serde_json::from_str(line).unwrap())
4028 .collect();
4029
4030 assert_eq!(values, vec![serde_json::json!({"id": 1}), serde_json::json!({"id": 2})]);
4031 }
4032
4033 #[test]
4034 fn canonical_double_wildcard_expands_a_root_array() {
4035 let input = br#"[{"id":1},{"id":2}]"#;
4036 let path = [JsonPathSegment::ArrayWildcard, JsonPathSegment::ArrayWildcard];
4037
4038 let result = flatten_json_document_to_ndjson(input, &path).expect("root wildcard should succeed");
4039
4040 assert_eq!(result, Bytes::from_static(b"{\"id\":1}\n{\"id\":2}\n"));
4041 }
4042
4043 #[test]
4044 fn terminal_scalar_path_uses_the_terminal_key_as_its_column() {
4045 let input = br#"{"rules":[{"id":"one"},{"id":"two"}]}"#;
4046 let path = [
4047 JsonPathSegment::ArrayWildcard,
4048 source_key("rules"),
4049 JsonPathSegment::ArrayWildcard,
4050 source_key("id"),
4051 ];
4052
4053 let result = flatten_json_document_to_ndjson(input, &path).expect("scalar source path should succeed");
4054
4055 assert_eq!(result, Bytes::from_static(b"{\"id\":\"one\"}\n{\"id\":\"two\"}\n"));
4056 }
4057
4058 #[test]
4059 fn terminal_scalar_path_uses_the_explicit_source_alias() {
4060 let input = br#"{"rules":[{"id":"one"},{"id":"two"}]}"#;
4061 let source = JsonSource::new(
4062 vec![
4063 JsonPathSegment::ArrayWildcard,
4064 source_key("rules"),
4065 JsonPathSegment::ArrayWildcard,
4066 source_key("id"),
4067 ],
4068 Some("v".to_string()),
4069 );
4070
4071 let result =
4072 super::parse_json_document_to_lines(input, &source).expect("explicit scalar source alias should be preserved");
4073
4074 assert_eq!(
4075 result,
4076 vec![
4077 Bytes::from_static(b"{\"v\":\"one\"}\n"),
4078 Bytes::from_static(b"{\"v\":\"two\"}\n")
4079 ]
4080 );
4081 }
4082
4083 #[test]
4084 fn source_keys_follow_s3_case_sensitivity_rules() {
4085 let input = br#"{"Employees":[{"id":1}]}"#;
4086
4087 let unquoted = flatten_json_document_to_ndjson(input, &[source_key("employees")])
4088 .expect("unquoted source key should be case insensitive");
4089 let quoted_exact = flatten_json_document_to_ndjson(input, &[quoted_source_key("Employees")])
4090 .expect("exact quoted source key should match");
4091 let quoted = flatten_json_document_to_ndjson(input, &[quoted_source_key("employees")])
4092 .expect("missing quoted source key should not fail");
4093
4094 assert_eq!(unquoted, Bytes::from_static(b"{\"id\":1}\n"));
4095 assert_eq!(quoted_exact, Bytes::from_static(b"{\"id\":1}\n"));
4096 assert!(quoted.is_empty());
4097 }
4098
4099 #[test]
4100 fn ambiguous_unquoted_source_key_is_rejected() {
4101 let input = br#"{"Employees":[],"employees":[]}"#;
4102
4103 let error =
4104 flatten_json_document_to_ndjson(input, &[source_key("EMPLOYEES")]).expect_err("ambiguous source key should fail");
4105
4106 assert!(
4107 error
4108 .get_ref()
4109 .and_then(|source| source.downcast_ref::<SelectError>())
4110 .is_some_and(|error| *error == SelectError::AmbiguousFieldName)
4111 );
4112 }
4113
4114 #[test]
4115 fn unquoted_source_key_scan_honors_cancellation() {
4116 let mut object = serde_json::Map::new();
4117 for index in 0..=super::JSON_CANCELLATION_CHECK_KEYS {
4118 object.insert(format!("field-{index}"), serde_json::Value::Null);
4119 }
4120 let mut checkpoints = 0;
4121 let mut cancel_on_second_checkpoint = || {
4122 checkpoints += 1;
4123 if checkpoints == 2 {
4124 Err(std::io::Error::new(std::io::ErrorKind::Interrupted, SelectError::Canceled))
4125 } else {
4126 Ok(())
4127 }
4128 };
4129
4130 let error = remove_json_source_key_with_checkpoint(&mut object, "x", false, &mut cancel_on_second_checkpoint)
4131 .expect_err("key scan should stop at its second cancellation checkpoint");
4132
4133 assert_eq!(error.kind(), std::io::ErrorKind::Interrupted);
4134 assert_eq!(checkpoints, 2);
4135 assert!(
4136 error
4137 .get_ref()
4138 .and_then(|source| source.downcast_ref::<SelectError>())
4139 .is_some_and(|error| *error == SelectError::Canceled)
4140 );
4141 }
4142
4143 #[test]
4144 fn long_json_key_comparison_honors_cancellation() {
4145 let key = "x".repeat(super::JSON_CANCELLATION_CHECK_BYTES * 2);
4146 let mut checkpoints = 0;
4147 let mut cancel_on_second_checkpoint = || {
4148 checkpoints += 1;
4149 if checkpoints == 2 {
4150 Err(std::io::Error::new(std::io::ErrorKind::Interrupted, SelectError::Canceled))
4151 } else {
4152 Ok(())
4153 }
4154 };
4155
4156 let error = json_key_eq_ignore_ascii_case_with_checkpoint(&key, &key, &mut cancel_on_second_checkpoint)
4157 .expect_err("long-key comparison should stop at its second cancellation checkpoint");
4158
4159 assert_eq!(error.kind(), std::io::ErrorKind::Interrupted);
4160 assert_eq!(checkpoints, 2);
4161 }
4162
4163 #[test]
4164 fn missing_source_key_produces_no_records() {
4165 let input = br#"{"employees":[]}"#;
4166
4167 let result = flatten_json_document_to_ndjson(input, &[source_key("nonexistent")])
4168 .expect("missing source key should not fail the query");
4169
4170 assert!(result.is_empty());
4171 }
4172
4173 #[tokio::test]
4174 async fn source_path_type_mismatches_are_json_parsing_errors() {
4175 let cases: [(&str, &[u8], Vec<JsonPathSegment>); 4] = [
4176 ("key on array", b"[1]", vec![source_key("id")]),
4177 ("index on object", b"{}", vec![JsonPathSegment::Index(0)]),
4178 (
4179 "array wildcard on object",
4180 b"{}",
4181 vec![JsonPathSegment::ArrayWildcard, JsonPathSegment::ArrayWildcard],
4182 ),
4183 ("object wildcard on array", b"[1]", vec![JsonPathSegment::ObjectWildcard]),
4184 ];
4185
4186 for (case, input, path) in cases {
4187 let memory_pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(JSON_DOCUMENT_TEST_MEMORY_BYTES));
4188 let mut output = json_document_ndjson_stream(
4189 Box::new(std::io::Cursor::new(input.to_vec())),
4190 input.len() as u64,
4191 JsonSource::from_path(path),
4192 Arc::new(SelectInputMetrics::default()),
4193 memory_pool,
4194 None,
4195 )
4196 .expect("build source-path JSON document stream");
4197 let source = output
4198 .next()
4199 .await
4200 .unwrap_or_else(|| panic!("{case} should produce one stream error"))
4201 .unwrap_err();
4202 let error = QueryError::from(DataFusionError::ObjectStore(Box::new(source)));
4203
4204 assert_eq!(error.select_error(), SelectError::JsonParsingError, "{case}");
4205 assert!(output.next().await.is_none(), "{case}");
4206 }
4207 }
4208}