1use crate::query::{Context, Query, ast::JsonSource};
16use crate::{
17 QueryError, QueryResult, SelectInputMetrics, SelectObjectSnapshot,
18 object_store::{EcObjectStore, is_json_document_input, legacy_json_source_from_input},
19};
20use datafusion::{
21 arrow::{
22 array::{Int32Array, StringArray},
23 datatypes::{DataType, Field, Schema},
24 record_batch::RecordBatch,
25 },
26 common::DataFusionError,
27 execution::{SessionStateBuilder, config::SessionConfig, context::SessionState, runtime_env::RuntimeEnvBuilder},
28 object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path},
29 parquet::arrow::ArrowWriter,
30 prelude::SessionContext,
31};
32use parking_lot::Mutex;
33use s3s::dto::CompressionType;
34use std::sync::{
35 Arc, Weak,
36 atomic::{AtomicU8, Ordering},
37};
38use tokio::{
39 sync::OwnedSemaphorePermit,
40 task::AbortHandle,
41 time::{Instant, sleep_until},
42};
43use tracing::error;
44
45pub type QueryExecutionGuard = Arc<OwnedSemaphorePermit>;
46
47pub struct QueryAdmission {
50 query_guard: Option<QueryExecutionGuard>,
51}
52
53impl QueryAdmission {
54 pub fn new(query_guard: QueryExecutionGuard) -> Self {
55 Self {
56 query_guard: Some(query_guard),
57 }
58 }
59
60 pub fn into_query_guard(mut self) -> Option<QueryExecutionGuard> {
61 self.query_guard.take()
62 }
63
64 pub(crate) fn unmanaged() -> Self {
65 Self { query_guard: None }
66 }
67}
68
69#[derive(Clone, Default)]
70pub struct QueryExecutionOwner {
71 identity: Arc<()>,
72}
73
74impl QueryExecutionOwner {
75 pub fn new() -> Self {
76 Self { identity: Arc::new(()) }
77 }
78}
79
80#[derive(Clone)]
81pub struct QueryExecutionTracker {
82 inner: Arc<QueryExecutionTrackerInner>,
83}
84
85#[derive(Clone, Copy, Debug, Eq, PartialEq)]
86pub enum QueryExecutionStatus {
87 Active,
88 Finished,
89 TimedOut,
90}
91
92const EXECUTION_SETTING_UP: u8 = 0;
93const EXECUTION_ADMITTED: u8 = 1;
94const EXECUTION_PLANNING: u8 = 2;
95const EXECUTION_PLANNED: u8 = 3;
96const EXECUTION_STARTING: u8 = 4;
97const EXECUTION_RUNNING: u8 = 5;
98const EXECUTION_FINISHED: u8 = 6;
99const EXECUTION_TIMED_OUT: u8 = 7;
100
101struct QueryExecutionTrackerInner {
102 owner_identity: Arc<()>,
103 query_guard: Mutex<Option<QueryExecutionGuard>>,
104 deadline: Instant,
105 timeout_seconds: u64,
106 state: AtomicU8,
107 deadline_task: Mutex<Option<AbortHandle>>,
108}
109
110impl QueryExecutionTracker {
111 pub fn new(owner: &QueryExecutionOwner, query_guard: QueryExecutionGuard, deadline: Instant, timeout_seconds: u64) -> Self {
112 let inner = Arc::new(QueryExecutionTrackerInner {
113 owner_identity: Arc::clone(&owner.identity),
114 query_guard: Mutex::new(Some(query_guard)),
115 deadline,
116 timeout_seconds,
117 state: AtomicU8::new(EXECUTION_SETTING_UP),
118 deadline_task: Mutex::new(None),
119 });
120 let deadline_inner = Arc::downgrade(&inner);
121 let deadline_task = tokio::spawn(async move {
122 sleep_until(deadline).await;
123 if let Some(inner) = Weak::upgrade(&deadline_inner) {
124 inner.expire_at_deadline();
125 }
126 });
127 *inner.deadline_task.lock() = Some(deadline_task.abort_handle());
128
129 Self { inner }
130 }
131
132 pub fn deadline(&self) -> Instant {
133 self.inner.deadline
134 }
135
136 pub fn timeout_seconds(&self) -> u64 {
137 self.inner.timeout_seconds
138 }
139
140 pub fn status(&self) -> QueryExecutionStatus {
141 match self.inner.state.load(Ordering::Acquire) {
142 EXECUTION_TIMED_OUT => QueryExecutionStatus::TimedOut,
143 state if state < EXECUTION_FINISHED => QueryExecutionStatus::Active,
144 _ => QueryExecutionStatus::Finished,
145 }
146 }
147
148 pub fn is_owned_by(&self, owner: &QueryExecutionOwner) -> bool {
149 Arc::ptr_eq(&self.inner.owner_identity, &owner.identity)
150 }
151
152 pub(crate) fn is_same_execution(&self, other: &Self) -> bool {
153 Arc::ptr_eq(&self.inner, &other.inner)
154 }
155
156 pub fn mark_admitted(&self, owner: &QueryExecutionOwner) -> bool {
157 self.transition(owner, EXECUTION_SETTING_UP, EXECUTION_ADMITTED)
158 }
159
160 pub fn claim_planning(&self, owner: &QueryExecutionOwner) -> bool {
161 self.transition(owner, EXECUTION_ADMITTED, EXECUTION_PLANNING)
162 }
163
164 pub fn mark_planned(&self, owner: &QueryExecutionOwner) -> bool {
165 self.transition(owner, EXECUTION_PLANNING, EXECUTION_PLANNED)
166 }
167
168 pub fn claim_execution(&self, owner: &QueryExecutionOwner) -> bool {
169 self.transition(owner, EXECUTION_PLANNED, EXECUTION_STARTING)
170 }
171
172 pub fn mark_running(&self, owner: &QueryExecutionOwner) -> bool {
173 self.transition(owner, EXECUTION_STARTING, EXECUTION_RUNNING)
174 }
175
176 pub fn handoff_deadline(&self, owner: &QueryExecutionOwner) {
177 if !self.is_owned_by(owner) || self.inner.state.load(Ordering::Acquire) != EXECUTION_RUNNING {
178 return;
179 }
180 if let Some(deadline_task) = self.inner.deadline_task.lock().take() {
181 deadline_task.abort();
182 }
183 }
184
185 pub fn finish(&self, owner: &QueryExecutionOwner) {
186 if self.is_owned_by(owner) {
187 self.inner.finish();
188 }
189 }
190
191 pub fn expire(&self, owner: &QueryExecutionOwner) {
192 if self.is_owned_by(owner) {
193 self.inner.expire();
194 }
195 }
196
197 pub(crate) fn query_guard(&self) -> Option<QueryExecutionGuard> {
198 let query_guard = self.inner.query_guard.lock();
199 if Instant::now() >= self.inner.deadline || self.status() != QueryExecutionStatus::Active {
200 return None;
201 }
202 query_guard.clone()
203 }
204
205 fn transition(&self, owner: &QueryExecutionOwner, from: u8, to: u8) -> bool {
206 self.is_owned_by(owner)
207 && self
208 .inner
209 .state
210 .compare_exchange(from, to, Ordering::AcqRel, Ordering::Acquire)
211 .is_ok()
212 }
213}
214
215impl QueryExecutionTrackerInner {
216 fn finish(&self) {
217 self.state.fetch_max(EXECUTION_FINISHED, Ordering::AcqRel);
218 self.release();
219 }
220
221 fn expire(&self) {
222 self.mark_timed_out();
223 self.release();
224 }
225
226 fn expire_at_deadline(&self) {
227 if self
228 .mark_timed_out()
229 .is_some_and(|state| matches!(state, EXECUTION_ADMITTED | EXECUTION_PLANNED))
230 {
231 self.release();
232 }
233 }
234
235 fn mark_timed_out(&self) -> Option<u8> {
236 self.state
237 .try_update(Ordering::AcqRel, Ordering::Acquire, |state| {
238 (state < EXECUTION_FINISHED).then_some(EXECUTION_TIMED_OUT)
239 })
240 .ok()
241 }
242
243 fn release(&self) {
244 self.query_guard.lock().take();
245 if let Some(deadline_task) = self.deadline_task.lock().take() {
246 deadline_task.abort();
247 }
248 }
249}
250
251impl Drop for QueryExecutionTrackerInner {
252 fn drop(&mut self) {
253 if let Some(deadline_task) = self.deadline_task.get_mut().take() {
254 deadline_task.abort();
255 }
256 }
257}
258
259impl std::fmt::Debug for QueryExecutionTracker {
260 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
261 f.debug_struct("QueryExecutionTracker")
262 .field("deadline", &self.deadline())
263 .field("timeout_seconds", &self.timeout_seconds())
264 .field("status", &self.status())
265 .finish_non_exhaustive()
266 }
267}
268
269#[derive(Clone)]
270pub struct SessionCtx {
271 _desc: Arc<SessionCtxDesc>,
272 inner: SessionState,
273 query_tracker: Option<QueryExecutionTracker>,
274}
275
276impl SessionCtx {
277 pub fn inner(&self) -> &SessionState {
278 &self.inner
279 }
280
281 pub(crate) fn is_bound_to(&self, query_tracker: &QueryExecutionTracker) -> bool {
282 self.query_tracker
283 .as_ref()
284 .is_some_and(|bound_tracker| bound_tracker.is_same_execution(query_tracker))
285 }
286}
287
288#[derive(Clone)]
289pub struct SessionCtxDesc {
290 }
292
293pub struct SessionCtxFactory {
294 pub is_test: bool,
295 pub target_partitions: usize,
296}
297
298pub const DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES: usize = 64 * 1024 * 1024;
299
300impl Default for SessionCtxFactory {
301 fn default() -> Self {
302 Self::new(false)
303 }
304}
305
306impl SessionCtxFactory {
307 pub fn new(is_test: bool) -> Self {
308 Self {
309 is_test,
310 target_partitions: 0,
311 }
312 }
313
314 pub fn with_target_partitions(mut self, target_partitions: usize) -> Self {
315 self.target_partitions = target_partitions;
316 self
317 }
318
319 pub async fn create_session_ctx(&self, context: &Context) -> QueryResult<SessionCtx> {
320 self.create_session_ctx_inner(
321 context,
322 None,
323 legacy_json_source_from_input(&context.input),
324 None,
325 None,
326 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
327 )
328 .await
329 }
330
331 pub async fn create_session_ctx_with_tracker_and_memory_limit(
332 &self,
333 context: &Context,
334 query_tracker: QueryExecutionTracker,
335 memory_limit_bytes: usize,
336 ) -> QueryResult<SessionCtx> {
337 self.create_session_ctx_inner(
338 context,
339 None,
340 legacy_json_source_from_input(&context.input),
341 Some(query_tracker),
342 None,
343 memory_limit_bytes,
344 )
345 .await
346 }
347
348 pub async fn create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
349 &self,
350 context: &Context,
351 snapshot: Arc<SelectObjectSnapshot>,
352 query_tracker: QueryExecutionTracker,
353 memory_limit_bytes: usize,
354 ) -> QueryResult<SessionCtx> {
355 self.create_session_ctx_inner(
356 context,
357 Some(snapshot),
358 legacy_json_source_from_input(&context.input),
359 Some(query_tracker),
360 None,
361 memory_limit_bytes,
362 )
363 .await
364 }
365
366 pub async fn create_session_ctx_for_query_with_source_and_tracker_and_memory_limit(
367 &self,
368 query: &Query,
369 source: JsonSource,
370 query_tracker: QueryExecutionTracker,
371 memory_limit_bytes: usize,
372 ) -> QueryResult<SessionCtx> {
373 self.create_session_ctx_inner(
374 query.context(),
375 query.snapshot().cloned(),
376 source,
377 Some(query_tracker),
378 Some(Arc::clone(query.input_metrics())),
379 memory_limit_bytes,
380 )
381 .await
382 }
383
384 pub async fn create_session_ctx_for_query_with_tracker_and_memory_limit(
385 &self,
386 query: &Query,
387 query_tracker: QueryExecutionTracker,
388 memory_limit_bytes: usize,
389 ) -> QueryResult<SessionCtx> {
390 self.create_session_ctx_inner(
391 query.context(),
392 query.snapshot().cloned(),
393 legacy_json_source_from_input(&query.context().input),
394 Some(query_tracker),
395 Some(Arc::clone(query.input_metrics())),
396 memory_limit_bytes,
397 )
398 .await
399 }
400
401 async fn create_session_ctx_inner(
402 &self,
403 context: &Context,
404 snapshot: Option<Arc<SelectObjectSnapshot>>,
405 source: JsonSource,
406 query_tracker: Option<QueryExecutionTracker>,
407 input_metrics: Option<Arc<SelectInputMetrics>>,
408 memory_limit_bytes: usize,
409 ) -> QueryResult<SessionCtx> {
410 let df_session_ctx = self
411 .build_df_session_context(context, snapshot, source, query_tracker.clone(), input_metrics, memory_limit_bytes)
412 .await?;
413
414 Ok(SessionCtx {
415 _desc: Arc::new(SessionCtxDesc {}),
416 inner: df_session_ctx.state(),
417 query_tracker,
418 })
419 }
420
421 async fn build_df_session_context(
422 &self,
423 context: &Context,
424 snapshot: Option<Arc<SelectObjectSnapshot>>,
425 source: JsonSource,
426 query_tracker: Option<QueryExecutionTracker>,
427 input_metrics: Option<Arc<SelectInputMetrics>>,
428 memory_limit_bytes: usize,
429 ) -> QueryResult<SessionContext> {
430 let path = format!("s3://{}", context.input.bucket);
431 let store_url = url::Url::parse(&path).unwrap();
432 let memory_limit_bytes = if memory_limit_bytes == 0 {
433 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES
434 } else {
435 memory_limit_bytes
436 };
437 let rt = RuntimeEnvBuilder::new().with_memory_limit(memory_limit_bytes, 1.0).build()?;
438 let config = SessionConfig::new().with_target_partitions(self.target_partitions);
439 let custom_two_byte_record_delimiter = context
440 .input
441 .request
442 .input_serialization
443 .csv
444 .as_ref()
445 .and_then(|csv| csv.record_delimiter.as_deref())
446 .is_some_and(|delimiter| delimiter.len() == 2 && delimiter.as_bytes() != b"\r\n");
447 let scan_range_requires_single_file_scan =
448 context.input.request.scan_range.is_some() && context.input.request.input_serialization.parquet.is_none();
449 let json_document_requires_single_file_scan = is_json_document_input(&context.input);
450 let compressed_input_requires_single_file_scan = context
451 .input
452 .request
453 .input_serialization
454 .compression_type
455 .as_ref()
456 .is_some_and(|compression| compression.as_str() != CompressionType::NONE);
457 let metered_input_requires_single_file_scan =
458 input_metrics.is_some() && context.input.request.input_serialization.parquet.is_none();
459 let normalized_csv_requires_single_file_scan =
460 context.input.request.input_serialization.csv.as_ref().is_some_and(|csv| {
461 crate::csv_input_requires_normalization(csv.quote_character.as_deref(), csv.quote_escape_character.as_deref())
462 });
463 let config = if normalized_csv_requires_single_file_scan
464 || custom_two_byte_record_delimiter
465 || scan_range_requires_single_file_scan
466 || json_document_requires_single_file_scan
467 || compressed_input_requires_single_file_scan
468 || metered_input_requires_single_file_scan
469 {
470 config.with_repartition_file_scans(false)
471 } else {
472 config
473 };
474 let memory_pool = Arc::clone(&rt.memory_pool);
475 let df_session_state = SessionStateBuilder::new()
476 .with_config(config)
477 .with_runtime_env(Arc::new(rt))
478 .with_default_features();
479
480 let df_session_state = if self.is_test {
481 let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
482
483 let data_bytes: Vec<u8> = if context.input.request.input_serialization.parquet.is_some() {
485 test_parquet_bytes()?
486 } else if context.input.request.input_serialization.json.is_some() {
487 b"{\"id\":1,\"name\":\"Alice\",\"age\":25,\"department\":\"HR\",\"salary\":5000}\n\
491 {\"id\":2,\"name\":\"Bob\",\"age\":30,\"department\":\"IT\",\"salary\":6000}\n\
492 {\"id\":3,\"name\":\"Charlie\",\"age\":35,\"department\":\"Finance\",\"salary\":7000}\n\
493 {\"id\":4,\"name\":\"Diana\",\"age\":22,\"department\":\"Marketing\",\"salary\":4500}\n\
494 {\"id\":5,\"name\":\"Eve\",\"age\":28,\"department\":\"IT\",\"salary\":5500}\n\
495 {\"id\":6,\"name\":\"Frank\",\"age\":40,\"department\":\"Finance\",\"salary\":8000}\n\
496 {\"id\":7,\"name\":\"Grace\",\"age\":26,\"department\":\"HR\",\"salary\":5200}\n\
497 {\"id\":8,\"name\":\"Henry\",\"age\":32,\"department\":\"IT\",\"salary\":6200}\n\
498 {\"id\":9,\"name\":\"Ivy\",\"age\":24,\"department\":\"Marketing\",\"salary\":4800}\n\
499 {\"id\":10,\"name\":\"Jack\",\"age\":38,\"department\":\"Finance\",\"salary\":7500}\n"
500 .to_vec()
501 } else {
502 b"id,name,age,department,salary\n\
503 1,Alice,25,HR,05000\n\
504 2,Bob,30,IT,6000\n\
505 3,Charlie,35,Finance,7000\n\
506 4,Diana,22,Marketing,4500\n\
507 5,Eve,28,IT,5500\n\
508 6,Frank,40,Finance,8000\n\
509 7,Grace,26,HR,5200\n\
510 8,Henry,32,IT,6200\n\
511 9,Ivy,24,Marketing,4800\n\
512 10,Jack,38,Finance,7500"
513 .to_vec()
514 };
515
516 let path = Path::from(context.input.key.clone());
517 store.put(&path, data_bytes.into()).await.map_err(|e| {
518 error!("put data into memory failed: {}", e.to_string());
519 QueryError::from(DataFusionError::from(e))
520 })?;
521
522 df_session_state.with_object_store(&store_url, store).build()
523 } else {
524 let input_metrics = input_metrics.unwrap_or_else(|| Arc::new(SelectInputMetrics::default()));
525 let store: EcObjectStore = match query_tracker {
526 Some(query_tracker) => EcObjectStore::new_with_query_tracker_and_source(
527 context.input.clone(),
528 memory_pool,
529 query_tracker,
530 input_metrics,
531 snapshot,
532 source,
533 ),
534 None => EcObjectStore::new_with_memory_pool_and_source(
535 context.input.clone(),
536 memory_pool,
537 input_metrics,
538 snapshot,
539 source,
540 ),
541 }
542 .map_err(|err| QueryError::Datafusion {
543 source: Box::new(DataFusionError::External(Box::new(err))),
544 })?;
545 df_session_state.with_object_store(&store_url, Arc::new(store)).build()
546 };
547
548 let df_session_ctx = SessionContext::new_with_state(df_session_state);
549
550 Ok(df_session_ctx)
551 }
552}
553
554fn test_parquet_bytes() -> QueryResult<Vec<u8>> {
555 let schema = Arc::new(Schema::new(vec![
556 Field::new("id", DataType::Int32, false),
557 Field::new("name", DataType::Utf8, false),
558 Field::new("age", DataType::Int32, false),
559 Field::new("department", DataType::Utf8, false),
560 Field::new("salary", DataType::Int32, false),
561 ]));
562 let first_batch =
563 test_parquet_batch(Arc::clone(&schema), &[1, 2], &["Alice", "Bob"], &[25, 30], &["HR", "IT"], &[5000, 6000])?;
564 let second_batch = test_parquet_batch(
565 Arc::clone(&schema),
566 &[3, 4, 5],
567 &["Charlie", "Diana", "Eve"],
568 &[35, 22, 28],
569 &["Finance", "Marketing", "IT"],
570 &[7000, 4500, 5500],
571 )?;
572
573 let mut bytes = Vec::new();
574 {
575 let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).map_err(DataFusionError::from)?;
576 writer.write(&first_batch).map_err(DataFusionError::from)?;
577 writer.flush().map_err(DataFusionError::from)?;
578 writer.write(&second_batch).map_err(DataFusionError::from)?;
579 writer.close().map_err(DataFusionError::from)?;
580 }
581 Ok(bytes)
582}
583
584fn test_parquet_batch(
585 schema: Arc<Schema>,
586 ids: &[i32],
587 names: &[&str],
588 ages: &[i32],
589 departments: &[&str],
590 salaries: &[i32],
591) -> QueryResult<RecordBatch> {
592 RecordBatch::try_new(
593 schema,
594 vec![
595 Arc::new(Int32Array::from(ids.to_vec())),
596 Arc::new(StringArray::from(names.to_vec())),
597 Arc::new(Int32Array::from(ages.to_vec())),
598 Arc::new(StringArray::from(departments.to_vec())),
599 Arc::new(Int32Array::from(salaries.to_vec())),
600 ],
601 )
602 .map_err(DataFusionError::from)
603 .map_err(QueryError::from)
604}
605
606#[cfg(test)]
607mod tests {
608 use super::*;
609 use crate::storage_api::SelectPutObjReader;
610 use crate::storage_api::object_store::ObjectIO as _;
611 use datafusion::{
612 datasource::{
613 file_format::{csv::CsvFormat, json::JsonFormat},
614 listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
615 },
616 execution::memory_pool::MemoryLimit,
617 };
618 use http::HeaderMap;
619 use s3s::dto::{
620 CSVInput, CSVOutput, ExpressionType, InputSerialization, JSONInput, JSONType, OutputSerialization, ParquetInput,
621 ScanRange, SelectObjectContentInput, SelectObjectContentRequest,
622 };
623 use std::io::Write as _;
624
625 fn test_context() -> Context {
626 Context {
627 input: Arc::new(SelectObjectContentInput {
628 bucket: "test-bucket".to_string(),
629 expected_bucket_owner: None,
630 key: "test.csv".to_string(),
631 sse_customer_algorithm: None,
632 sse_customer_key: None,
633 sse_customer_key_md5: None,
634 request: SelectObjectContentRequest {
635 expression: "SELECT * FROM S3Object".to_string(),
636 expression_type: ExpressionType::from_static("SQL"),
637 input_serialization: InputSerialization {
638 csv: Some(CSVInput::default()),
639 ..Default::default()
640 },
641 output_serialization: OutputSerialization {
642 csv: Some(CSVOutput::default()),
643 ..Default::default()
644 },
645 request_progress: None,
646 scan_range: None,
647 },
648 }),
649 }
650 }
651
652 async fn prepare_test_snapshot(context: &Context) -> Arc<SelectObjectSnapshot> {
653 let env = crate::storage_api::select_test_ecstore_env().await;
654 Arc::new(
655 env.ecstore
656 .prepare_select_object_snapshot(&context.input.bucket, &context.input.key, &HeaderMap::new(), &Default::default())
657 .await
658 .expect("prepare SelectObjectContent snapshot"),
659 )
660 }
661
662 async fn test_query_tracker() -> QueryExecutionTracker {
663 let permit = Arc::new(tokio::sync::Semaphore::new(1))
664 .acquire_owned()
665 .await
666 .expect("query permit should be available");
667 QueryExecutionTracker::new(
668 &QueryExecutionOwner::new(),
669 Arc::new(permit),
670 Instant::now() + std::time::Duration::from_secs(300),
671 300,
672 )
673 }
674
675 async fn assert_legacy_json_column(bucket: &str, expression: &str, document: &[u8], column_name: &str, expected: &[&str]) {
676 const OBJECT: &str = "input.json";
677
678 let env = crate::storage_api::select_test_ecstore_env().await;
679 let mut context = test_context();
680 {
681 let input = Arc::make_mut(&mut context.input);
682 input.bucket = bucket.to_string();
683 input.key = OBJECT.to_string();
684 input.request.expression = expression.to_string();
685 input.request.input_serialization.csv = None;
686 input.request.input_serialization.json = Some(JSONInput {
687 type_: Some(JSONType::from_static(JSONType::DOCUMENT)),
688 });
689 }
690 env.make_bucket(bucket, false).await;
691 env.put_object_bytes(bucket, OBJECT, document.to_vec()).await;
692 let factory = SessionCtxFactory::new(false);
693 let lazy_session = factory
694 .create_session_ctx(&context)
695 .await
696 .expect("legacy lazy session should preserve the JSON source");
697 let tracked_lazy_session = factory
698 .create_session_ctx_with_tracker_and_memory_limit(
699 &context,
700 test_query_tracker().await,
701 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
702 )
703 .await
704 .expect("legacy tracked lazy session should preserve the JSON source");
705 let snapshot = prepare_test_snapshot(&context).await;
706 let snapshot_session = factory
707 .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
708 &context,
709 snapshot,
710 test_query_tracker().await,
711 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
712 )
713 .await
714 .expect("legacy snapshot session should preserve the JSON source");
715
716 for (kind, session) in [
717 ("lazy", lazy_session),
718 ("tracked lazy", tracked_lazy_session),
719 ("tracked snapshot", snapshot_session),
720 ] {
721 let table_path = ListingTableUrl::parse(format!("s3://{bucket}/{OBJECT}")).expect("parse JSON table URL");
722 let listing_options = ListingOptions::new(Arc::new(JsonFormat::default())).with_file_extension(".json");
723 let schema = listing_options
724 .infer_schema(session.inner(), &table_path)
725 .await
726 .expect("infer expanded JSON schema");
727 let table = ListingTable::try_new(
728 ListingTableConfig::new(table_path)
729 .with_listing_options(listing_options)
730 .with_schema(schema),
731 )
732 .expect("build expanded JSON table");
733 let query_context = SessionContext::new_with_state(session.inner().clone());
734 query_context
735 .register_table("legacy_input", Arc::new(table))
736 .expect("register expanded JSON table");
737 let batches = query_context
738 .sql(&format!("SELECT {column_name} FROM legacy_input"))
739 .await
740 .expect("plan expanded JSON query")
741 .collect()
742 .await
743 .expect("execute expanded JSON query");
744 let mut values = Vec::new();
745 for batch in batches {
746 let column = batch
747 .column(0)
748 .as_any()
749 .downcast_ref::<StringArray>()
750 .expect("expanded column should be Utf8");
751 for row in 0..batch.num_rows() {
752 values.push(column.value(row).to_string());
753 }
754 }
755 assert_eq!(values, expected, "{kind} legacy constructor");
756 }
757 }
758
759 #[test]
760 fn session_factory_fields_remain_source_compatible() {
761 let factory = SessionCtxFactory {
762 is_test: true,
763 target_partitions: 0,
764 };
765
766 assert!(factory.is_test);
767 assert_eq!(factory.target_partitions, 0);
768 }
769
770 #[tokio::test]
771 async fn session_factory_applies_target_partitions() {
772 let factory = SessionCtxFactory::new(true).with_target_partitions(3);
773 let session = factory
774 .create_session_ctx(&test_context())
775 .await
776 .expect("session should be created with configured target partitions");
777
778 assert_eq!(session.inner().config().target_partitions(), 3);
779 assert!(session.inner().config().options().optimizer.repartition_file_scans);
780 }
781
782 #[tokio::test]
783 async fn metered_csv_and_json_inputs_disable_file_repartitioning() {
784 let factory = SessionCtxFactory::new(true).with_target_partitions(3);
785 let csv_context = test_context();
786 let mut json_context = test_context();
787 let json_request = &mut Arc::make_mut(&mut json_context.input).request;
788 json_request.input_serialization.csv = None;
789 json_request.input_serialization.json = Some(JSONInput::default());
790
791 for context in [&csv_context, &json_context] {
792 let session = factory
793 .create_session_ctx_inner(
794 context,
795 None,
796 JsonSource::default(),
797 None,
798 Some(Arc::new(SelectInputMetrics::default())),
799 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
800 )
801 .await
802 .expect("metered session should be created");
803
804 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
805 }
806 }
807
808 #[tokio::test]
809 async fn parquet_scan_range_keeps_file_repartitioning() {
810 let mut context = test_context();
811 let request = &mut Arc::make_mut(&mut context.input).request;
812 request.scan_range = Some(ScanRange {
813 start: Some(0),
814 end: Some(0),
815 });
816 request.input_serialization.csv = None;
817 request.input_serialization.parquet = Some(ParquetInput {});
818
819 let session = SessionCtxFactory::new(true)
820 .with_target_partitions(2)
821 .create_session_ctx(&context)
822 .await
823 .expect("Parquet ScanRange session should be created");
824
825 assert!(session.inner().config().options().optimizer.repartition_file_scans);
826 }
827
828 #[tokio::test]
829 async fn json_lines_scan_range_disables_file_repartitioning() {
830 let mut context = test_context();
831 let request = &mut Arc::make_mut(&mut context.input).request;
832 request.scan_range = Some(ScanRange {
833 start: Some(0),
834 end: Some(0),
835 });
836 request.input_serialization.csv = None;
837 request.input_serialization.json = Some(JSONInput::default());
838
839 let session = SessionCtxFactory::new(true)
840 .with_target_partitions(2)
841 .create_session_ctx(&context)
842 .await
843 .expect("JSON LINES ScanRange session should be created");
844
845 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
846 }
847
848 #[tokio::test]
849 async fn json_lines_without_scan_range_keeps_file_repartitioning() {
850 let mut context = test_context();
851 let request = &mut Arc::make_mut(&mut context.input).request;
852 request.input_serialization.csv = None;
853 request.input_serialization.json = Some(JSONInput::default());
854
855 let session = SessionCtxFactory::new(true)
856 .with_target_partitions(2)
857 .create_session_ctx(&context)
858 .await
859 .expect("JSON LINES session should be created");
860
861 assert!(session.inner().config().options().optimizer.repartition_file_scans);
862 }
863
864 #[tokio::test]
865 async fn compressed_input_disables_file_repartitioning_without_metrics() {
866 let mut context = test_context();
867 Arc::make_mut(&mut context.input).request.input_serialization.compression_type =
868 Some(CompressionType::from_static(CompressionType::GZIP));
869
870 let session = SessionCtxFactory::new(true)
871 .with_target_partitions(2)
872 .create_session_ctx(&context)
873 .await
874 .expect("compressed session should be created");
875
876 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
877 }
878
879 #[tokio::test]
880 async fn json_document_disables_file_repartitioning() {
881 let mut context = test_context();
882 let request = &mut Arc::make_mut(&mut context.input).request;
883 request.input_serialization.csv = None;
884 request.input_serialization.json = Some(JSONInput {
885 type_: Some(JSONType::from_static(JSONType::DOCUMENT)),
886 });
887
888 let session = SessionCtxFactory::new(true)
889 .with_target_partitions(2)
890 .create_session_ctx(&context)
891 .await
892 .expect("JSON DOCUMENT session should be created");
893
894 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
895 }
896
897 #[tokio::test]
898 async fn csv_scan_range_disables_file_repartitioning() {
899 let mut context = test_context();
900 Arc::make_mut(&mut context.input).request.scan_range = Some(ScanRange {
901 start: Some(0),
902 end: Some(0),
903 });
904
905 let session = SessionCtxFactory::new(true)
906 .with_target_partitions(2)
907 .create_session_ctx(&context)
908 .await
909 .expect("CSV ScanRange session should be created");
910
911 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
912 }
913
914 #[tokio::test]
915 async fn unicode_csv_quotes_disable_file_scan_repartition() {
916 let mut context = test_context();
917 Arc::get_mut(&mut context.input)
918 .expect("unique context")
919 .request
920 .input_serialization
921 .csv
922 .as_mut()
923 .expect("CSV input")
924 .quote_character = Some("ع".to_owned());
925 let session = SessionCtxFactory::new(true)
926 .with_target_partitions(4)
927 .create_session_ctx(&context)
928 .await
929 .expect("Unicode CSV session");
930 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
931 }
932
933 #[tokio::test]
934 async fn two_byte_csv_record_delimiter_disables_file_scan_repartition() {
935 let mut context = test_context();
936 Arc::get_mut(&mut context.input)
937 .expect("test context input should be uniquely owned")
938 .request
939 .input_serialization
940 .csv
941 .as_mut()
942 .expect("test context should use CSV")
943 .record_delimiter = Some("^Y".to_string());
944 let session = SessionCtxFactory::new(true)
945 .with_target_partitions(4)
946 .create_session_ctx(&context)
947 .await
948 .expect("session should be created");
949
950 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
951 }
952
953 #[tokio::test]
954 async fn crlf_record_delimiter_retains_file_scan_repartition() {
955 let mut context = test_context();
956 Arc::get_mut(&mut context.input)
957 .expect("test context input should be uniquely owned")
958 .request
959 .input_serialization
960 .csv
961 .as_mut()
962 .expect("test context should use CSV")
963 .record_delimiter = Some("\r\n".to_string());
964 let session = SessionCtxFactory::new(true)
965 .with_target_partitions(4)
966 .create_session_ctx(&context)
967 .await
968 .expect("session should be created");
969
970 assert_eq!(
971 session.inner().config().options().optimizer.repartition_file_scans,
972 SessionConfig::new().options().optimizer.repartition_file_scans
973 );
974 }
975
976 #[tokio::test]
977 async fn session_factory_zero_target_partitions_uses_datafusion_default() {
978 let factory = SessionCtxFactory::new(true);
979 let session = factory
980 .create_session_ctx(&test_context())
981 .await
982 .expect("session should be created with default target partitions");
983
984 assert_eq!(session.inner().config().target_partitions(), SessionConfig::new().target_partitions());
985 }
986
987 #[tokio::test]
988 async fn session_factory_applies_memory_limit() {
989 let factory = SessionCtxFactory::new(true);
990 let session = factory
991 .create_session_ctx_inner(&test_context(), None, JsonSource::default(), None, None, 1024)
992 .await
993 .expect("session should be created with a bounded memory pool");
994
995 assert!(matches!(
996 session.inner().runtime_env().memory_pool.memory_limit(),
997 MemoryLimit::Finite(1024)
998 ));
999 }
1000
1001 #[tokio::test]
1002 #[serial_test::serial]
1003 async fn production_session_preserves_lazy_snapshot_entry() {
1004 let _env = crate::storage_api::select_test_ecstore_env().await;
1005 let session = SessionCtxFactory::new(false)
1006 .create_session_ctx(&test_context())
1007 .await
1008 .expect("legacy production session should install a lazy object store");
1009
1010 assert_eq!(session.inner().config().target_partitions(), SessionConfig::new().target_partitions());
1011 }
1012
1013 #[tokio::test]
1014 #[serial_test::serial]
1015 async fn legacy_tracked_production_session_preserves_lazy_snapshot_entry() {
1016 let _env = crate::storage_api::select_test_ecstore_env().await;
1017 let permit = Arc::new(tokio::sync::Semaphore::new(1))
1018 .acquire_owned()
1019 .await
1020 .expect("query permit should be available");
1021 let tracker = QueryExecutionTracker::new(
1022 &QueryExecutionOwner::new(),
1023 Arc::new(permit),
1024 Instant::now() + std::time::Duration::from_secs(300),
1025 300,
1026 );
1027 let session = SessionCtxFactory::new(false)
1028 .create_session_ctx_with_tracker_and_memory_limit(
1029 &test_context(),
1030 tracker.clone(),
1031 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1032 )
1033 .await
1034 .expect("legacy tracked session should install a lazy object store");
1035
1036 assert!(session.is_bound_to(&tracker));
1037 }
1038
1039 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1040 #[serial_test::serial]
1041 async fn legacy_session_factory_preserves_single_key_json_source() {
1042 assert_legacy_json_column(
1043 "s3select-legacy-session-json-source",
1044 "SELECT e.name FROM S3Object.employees AS e",
1045 br#"{"employees":[{"name":"Alice"}]}"#,
1046 "name",
1047 &["Alice"],
1048 )
1049 .await;
1050 }
1051
1052 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1053 #[serial_test::serial]
1054 async fn legacy_session_factory_preserves_implicit_root_alias() {
1055 assert_legacy_json_column(
1056 "s3select-legacy-session-root-alias",
1057 "SELECT S3Object FROM S3Object",
1058 br#"["one","two"]"#,
1059 "s3object",
1060 &["one", "two"],
1061 )
1062 .await;
1063 }
1064
1065 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1066 #[serial_test::serial]
1067 async fn legacy_session_factory_preserves_quoted_root_alias() {
1068 assert_legacy_json_column(
1069 "s3select-legacy-session-quoted-root-alias",
1070 "SELECT \"V\" FROM S3Object AS \"V\"",
1071 br#"["one","two"]"#,
1072 "\"V\"",
1073 &["one", "two"],
1074 )
1075 .await;
1076 }
1077
1078 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1079 #[serial_test::serial]
1080 async fn session_factory_propagates_query_guard_to_ec_store() {
1081 let env = crate::storage_api::select_test_ecstore_env().await;
1082 let mut context = test_context();
1083 Arc::make_mut(&mut context.input).bucket = "s3select-query-guard-snapshot".to_string();
1084 env.make_bucket(&context.input.bucket, false).await;
1085 let mut reader = SelectPutObjReader::from_vec(b"id,name\n1,Alice\n".to_vec());
1086 env.ecstore
1087 .put_object(&context.input.bucket, &context.input.key, &mut reader, &Default::default())
1088 .await
1089 .expect("put query guard fixture");
1090 let snapshot = prepare_test_snapshot(&context).await;
1091
1092 let admission = Arc::new(tokio::sync::Semaphore::new(1));
1093 let permit = Arc::clone(&admission)
1094 .acquire_owned()
1095 .await
1096 .expect("query permit should be available");
1097 let query_guard = Arc::new(permit);
1098 let query_tracker = QueryExecutionTracker::new(
1099 &QueryExecutionOwner::new(),
1100 Arc::clone(&query_guard),
1101 Instant::now() + std::time::Duration::from_secs(300),
1102 300,
1103 );
1104 let session = SessionCtxFactory::new(false)
1105 .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
1106 &context,
1107 snapshot,
1108 query_tracker,
1109 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1110 )
1111 .await
1112 .expect("production session should be created with the query guard");
1113
1114 assert!(Arc::strong_count(&query_guard) > 1);
1115 drop(session);
1116 assert_eq!(Arc::strong_count(&query_guard), 1);
1117 }
1118
1119 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1120 #[serial_test::serial]
1121 async fn session_factory_preserves_snapshot_binding_error_source() {
1122 let env = crate::storage_api::select_test_ecstore_env().await;
1123 let mut source_context = test_context();
1124 Arc::make_mut(&mut source_context.input).bucket = "s3select-session-snapshot-identity".to_string();
1125 Arc::make_mut(&mut source_context.input).key = "source.csv".to_string();
1126 env.make_bucket(&source_context.input.bucket, false).await;
1127 let mut reader = SelectPutObjReader::from_vec(b"source-marker\n".to_vec());
1128 env.ecstore
1129 .put_object(&source_context.input.bucket, &source_context.input.key, &mut reader, &Default::default())
1130 .await
1131 .expect("put snapshot identity fixture");
1132 let snapshot = prepare_test_snapshot(&source_context).await;
1133
1134 let mut target_context = source_context.clone();
1135 Arc::make_mut(&mut target_context.input).key = "different.csv".to_string();
1136 let permit = Arc::new(tokio::sync::Semaphore::new(1))
1137 .acquire_owned()
1138 .await
1139 .expect("query permit should be available");
1140 let tracker = QueryExecutionTracker::new(
1141 &QueryExecutionOwner::new(),
1142 Arc::new(permit),
1143 Instant::now() + std::time::Duration::from_secs(300),
1144 300,
1145 );
1146
1147 let error = match SessionCtxFactory::new(false)
1148 .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
1149 &target_context,
1150 snapshot,
1151 tracker,
1152 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1153 )
1154 .await
1155 {
1156 Ok(_) => panic!("a session must reject a snapshot for a different object"),
1157 Err(error) => error,
1158 };
1159
1160 assert!(error.is_snapshot_consistency_error());
1161 assert!(error.to_string().contains("snapshot consistency failure"));
1162 }
1163
1164 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1165 #[serial_test::serial]
1166 async fn scan_range_is_preserved_across_large_csv_partition_boundary() {
1167 const ROW_COUNT: usize = 200_000;
1168 const ROW_WIDTH: usize = 7;
1169 const SELECTED_ROW: usize = 100_000;
1170
1171 let env = crate::storage_api::select_test_ecstore_env().await;
1172 let mut data = Vec::with_capacity(ROW_COUNT * ROW_WIDTH);
1173 for row in 0..ROW_COUNT {
1174 writeln!(&mut data, "{row:06}").expect("write fixed-width CSV test row");
1175 }
1176 assert!(data.len() > 1024 * 1024);
1177
1178 let mut context = test_context();
1179 Arc::make_mut(&mut context.input).bucket = "s3select-scan-range-partition-snapshot".to_string();
1180 let selected_start = i64::try_from(SELECTED_ROW * ROW_WIDTH).expect("selected row offset should fit in i64");
1181 Arc::make_mut(&mut context.input).request.scan_range = Some(ScanRange {
1182 start: Some(selected_start),
1183 end: Some(selected_start),
1184 });
1185 env.make_bucket(&context.input.bucket, false).await;
1186 let mut reader = SelectPutObjReader::from_vec(data);
1187 env.ecstore
1188 .put_object(&context.input.bucket, &context.input.key, &mut reader, &Default::default())
1189 .await
1190 .expect("put large ScanRange CSV fixture");
1191 let snapshot = prepare_test_snapshot(&context).await;
1192
1193 let admission = Arc::new(tokio::sync::Semaphore::new(1));
1194 let permit = Arc::clone(&admission)
1195 .acquire_owned()
1196 .await
1197 .expect("query permit should be available");
1198 let query_tracker = QueryExecutionTracker::new(
1199 &QueryExecutionOwner::new(),
1200 Arc::new(permit),
1201 Instant::now() + std::time::Duration::from_secs(300),
1202 300,
1203 );
1204 let session = SessionCtxFactory::new(false)
1205 .with_target_partitions(2)
1206 .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
1207 &context,
1208 snapshot,
1209 query_tracker,
1210 DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1211 )
1212 .await
1213 .expect("create production ScanRange session");
1214 assert!(!session.inner().config().options().optimizer.repartition_file_scans);
1215
1216 let table_path = ListingTableUrl::parse(format!("s3://{}/{}", context.input.bucket, context.input.key))
1217 .expect("parse ScanRange table URL");
1218 let listing_options =
1219 ListingOptions::new(Arc::new(CsvFormat::default().with_schema_infer_max_rec(0).with_has_header(false)))
1220 .with_file_extension(".csv");
1221 let schema = listing_options
1222 .infer_schema(session.inner(), &table_path)
1223 .await
1224 .expect("infer ScanRange CSV schema");
1225 let table = ListingTable::try_new(
1226 ListingTableConfig::new(table_path)
1227 .with_listing_options(listing_options)
1228 .with_schema(schema),
1229 )
1230 .expect("build ScanRange listing table");
1231 let query_context = SessionContext::new_with_state(session.inner().clone());
1232 query_context
1233 .register_table("scan_input", Arc::new(table))
1234 .expect("register ScanRange listing table");
1235
1236 let batches = query_context
1237 .sql("SELECT * FROM scan_input")
1238 .await
1239 .expect("plan ScanRange CSV query")
1240 .collect()
1241 .await
1242 .expect("execute ScanRange CSV query");
1243 let mut values = Vec::new();
1244 for batch in batches {
1245 let column = batch
1246 .column(0)
1247 .as_any()
1248 .downcast_ref::<StringArray>()
1249 .expect("ScanRange CSV column should be Utf8");
1250 for row in 0..batch.num_rows() {
1251 values.push(column.value(row).to_string());
1252 }
1253 }
1254
1255 assert_eq!(values.len(), 1);
1256 assert_eq!(values, [format!("{SELECTED_ROW:06}")]);
1257 }
1258
1259 #[tokio::test]
1260 async fn elapsed_deadline_does_not_yield_query_guard_before_timer_poll() {
1261 let admission = Arc::new(tokio::sync::Semaphore::new(1));
1262 let permit = Arc::clone(&admission)
1263 .acquire_owned()
1264 .await
1265 .expect("query permit should be available");
1266 let query_tracker = QueryExecutionTracker::new(&QueryExecutionOwner::new(), Arc::new(permit), Instant::now(), 0);
1267
1268 assert!(query_tracker.query_guard().is_none());
1269 }
1270
1271 #[tokio::test]
1272 async fn session_factory_default_uses_bounded_memory() {
1273 let factory = SessionCtxFactory::default();
1274 let session = SessionCtxFactory::new(true)
1275 .create_session_ctx(&test_context())
1276 .await
1277 .expect("default session should be created with a bounded memory pool");
1278
1279 assert!(!factory.is_test);
1280 assert_eq!(factory.target_partitions, 0);
1281 assert!(matches!(
1282 session.inner().runtime_env().memory_pool.memory_limit(),
1283 MemoryLimit::Finite(DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES)
1284 ));
1285 }
1286}