Skip to main content

rustfs_s3select_api/query/
session.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::query::{Context, Query, ast::JsonSource};
16use crate::{
17    QueryError, QueryResult, SelectInputMetrics, SelectObjectSnapshot,
18    object_store::{EcObjectStore, is_json_document_input, legacy_json_source_from_input},
19};
20use datafusion::{
21    arrow::{
22        array::{Int32Array, StringArray},
23        datatypes::{DataType, Field, Schema},
24        record_batch::RecordBatch,
25    },
26    common::DataFusionError,
27    execution::{SessionStateBuilder, config::SessionConfig, context::SessionState, runtime_env::RuntimeEnvBuilder},
28    object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path},
29    parquet::arrow::ArrowWriter,
30    prelude::SessionContext,
31};
32use parking_lot::Mutex;
33use s3s::dto::CompressionType;
34use std::sync::{
35    Arc, Weak,
36    atomic::{AtomicU8, Ordering},
37};
38use tokio::{
39    sync::OwnedSemaphorePermit,
40    task::AbortHandle,
41    time::{Instant, sleep_until},
42};
43use tracing::error;
44
45pub type QueryExecutionGuard = Arc<OwnedSemaphorePermit>;
46
47/// A one-shot query admission reservation handed from the request boundary to
48/// the dispatcher that owns the corresponding concurrency semaphore.
49pub struct QueryAdmission {
50    query_guard: Option<QueryExecutionGuard>,
51}
52
53impl QueryAdmission {
54    pub fn new(query_guard: QueryExecutionGuard) -> Self {
55        Self {
56            query_guard: Some(query_guard),
57        }
58    }
59
60    pub fn into_query_guard(mut self) -> Option<QueryExecutionGuard> {
61        self.query_guard.take()
62    }
63
64    pub(crate) fn unmanaged() -> Self {
65        Self { query_guard: None }
66    }
67}
68
69#[derive(Clone, Default)]
70pub struct QueryExecutionOwner {
71    identity: Arc<()>,
72}
73
74impl QueryExecutionOwner {
75    pub fn new() -> Self {
76        Self { identity: Arc::new(()) }
77    }
78}
79
80#[derive(Clone)]
81pub struct QueryExecutionTracker {
82    inner: Arc<QueryExecutionTrackerInner>,
83}
84
85#[derive(Clone, Copy, Debug, Eq, PartialEq)]
86pub enum QueryExecutionStatus {
87    Active,
88    Finished,
89    TimedOut,
90}
91
92const EXECUTION_SETTING_UP: u8 = 0;
93const EXECUTION_ADMITTED: u8 = 1;
94const EXECUTION_PLANNING: u8 = 2;
95const EXECUTION_PLANNED: u8 = 3;
96const EXECUTION_STARTING: u8 = 4;
97const EXECUTION_RUNNING: u8 = 5;
98const EXECUTION_FINISHED: u8 = 6;
99const EXECUTION_TIMED_OUT: u8 = 7;
100
101struct QueryExecutionTrackerInner {
102    owner_identity: Arc<()>,
103    query_guard: Mutex<Option<QueryExecutionGuard>>,
104    deadline: Instant,
105    timeout_seconds: u64,
106    state: AtomicU8,
107    deadline_task: Mutex<Option<AbortHandle>>,
108}
109
110impl QueryExecutionTracker {
111    pub fn new(owner: &QueryExecutionOwner, query_guard: QueryExecutionGuard, deadline: Instant, timeout_seconds: u64) -> Self {
112        let inner = Arc::new(QueryExecutionTrackerInner {
113            owner_identity: Arc::clone(&owner.identity),
114            query_guard: Mutex::new(Some(query_guard)),
115            deadline,
116            timeout_seconds,
117            state: AtomicU8::new(EXECUTION_SETTING_UP),
118            deadline_task: Mutex::new(None),
119        });
120        let deadline_inner = Arc::downgrade(&inner);
121        let deadline_task = tokio::spawn(async move {
122            sleep_until(deadline).await;
123            if let Some(inner) = Weak::upgrade(&deadline_inner) {
124                inner.expire_at_deadline();
125            }
126        });
127        *inner.deadline_task.lock() = Some(deadline_task.abort_handle());
128
129        Self { inner }
130    }
131
132    pub fn deadline(&self) -> Instant {
133        self.inner.deadline
134    }
135
136    pub fn timeout_seconds(&self) -> u64 {
137        self.inner.timeout_seconds
138    }
139
140    pub fn status(&self) -> QueryExecutionStatus {
141        match self.inner.state.load(Ordering::Acquire) {
142            EXECUTION_TIMED_OUT => QueryExecutionStatus::TimedOut,
143            state if state < EXECUTION_FINISHED => QueryExecutionStatus::Active,
144            _ => QueryExecutionStatus::Finished,
145        }
146    }
147
148    pub fn is_owned_by(&self, owner: &QueryExecutionOwner) -> bool {
149        Arc::ptr_eq(&self.inner.owner_identity, &owner.identity)
150    }
151
152    pub(crate) fn is_same_execution(&self, other: &Self) -> bool {
153        Arc::ptr_eq(&self.inner, &other.inner)
154    }
155
156    pub fn mark_admitted(&self, owner: &QueryExecutionOwner) -> bool {
157        self.transition(owner, EXECUTION_SETTING_UP, EXECUTION_ADMITTED)
158    }
159
160    pub fn claim_planning(&self, owner: &QueryExecutionOwner) -> bool {
161        self.transition(owner, EXECUTION_ADMITTED, EXECUTION_PLANNING)
162    }
163
164    pub fn mark_planned(&self, owner: &QueryExecutionOwner) -> bool {
165        self.transition(owner, EXECUTION_PLANNING, EXECUTION_PLANNED)
166    }
167
168    pub fn claim_execution(&self, owner: &QueryExecutionOwner) -> bool {
169        self.transition(owner, EXECUTION_PLANNED, EXECUTION_STARTING)
170    }
171
172    pub fn mark_running(&self, owner: &QueryExecutionOwner) -> bool {
173        self.transition(owner, EXECUTION_STARTING, EXECUTION_RUNNING)
174    }
175
176    pub fn handoff_deadline(&self, owner: &QueryExecutionOwner) {
177        if !self.is_owned_by(owner) || self.inner.state.load(Ordering::Acquire) != EXECUTION_RUNNING {
178            return;
179        }
180        if let Some(deadline_task) = self.inner.deadline_task.lock().take() {
181            deadline_task.abort();
182        }
183    }
184
185    pub fn finish(&self, owner: &QueryExecutionOwner) {
186        if self.is_owned_by(owner) {
187            self.inner.finish();
188        }
189    }
190
191    pub fn expire(&self, owner: &QueryExecutionOwner) {
192        if self.is_owned_by(owner) {
193            self.inner.expire();
194        }
195    }
196
197    pub(crate) fn query_guard(&self) -> Option<QueryExecutionGuard> {
198        let query_guard = self.inner.query_guard.lock();
199        if Instant::now() >= self.inner.deadline || self.status() != QueryExecutionStatus::Active {
200            return None;
201        }
202        query_guard.clone()
203    }
204
205    fn transition(&self, owner: &QueryExecutionOwner, from: u8, to: u8) -> bool {
206        self.is_owned_by(owner)
207            && self
208                .inner
209                .state
210                .compare_exchange(from, to, Ordering::AcqRel, Ordering::Acquire)
211                .is_ok()
212    }
213}
214
215impl QueryExecutionTrackerInner {
216    fn finish(&self) {
217        self.state.fetch_max(EXECUTION_FINISHED, Ordering::AcqRel);
218        self.release();
219    }
220
221    fn expire(&self) {
222        self.mark_timed_out();
223        self.release();
224    }
225
226    fn expire_at_deadline(&self) {
227        if self
228            .mark_timed_out()
229            .is_some_and(|state| matches!(state, EXECUTION_ADMITTED | EXECUTION_PLANNED))
230        {
231            self.release();
232        }
233    }
234
235    fn mark_timed_out(&self) -> Option<u8> {
236        self.state
237            .try_update(Ordering::AcqRel, Ordering::Acquire, |state| {
238                (state < EXECUTION_FINISHED).then_some(EXECUTION_TIMED_OUT)
239            })
240            .ok()
241    }
242
243    fn release(&self) {
244        self.query_guard.lock().take();
245        if let Some(deadline_task) = self.deadline_task.lock().take() {
246            deadline_task.abort();
247        }
248    }
249}
250
251impl Drop for QueryExecutionTrackerInner {
252    fn drop(&mut self) {
253        if let Some(deadline_task) = self.deadline_task.get_mut().take() {
254            deadline_task.abort();
255        }
256    }
257}
258
259impl std::fmt::Debug for QueryExecutionTracker {
260    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
261        f.debug_struct("QueryExecutionTracker")
262            .field("deadline", &self.deadline())
263            .field("timeout_seconds", &self.timeout_seconds())
264            .field("status", &self.status())
265            .finish_non_exhaustive()
266    }
267}
268
269#[derive(Clone)]
270pub struct SessionCtx {
271    _desc: Arc<SessionCtxDesc>,
272    inner: SessionState,
273    query_tracker: Option<QueryExecutionTracker>,
274}
275
276impl SessionCtx {
277    pub fn inner(&self) -> &SessionState {
278        &self.inner
279    }
280
281    pub(crate) fn is_bound_to(&self, query_tracker: &QueryExecutionTracker) -> bool {
282        self.query_tracker
283            .as_ref()
284            .is_some_and(|bound_tracker| bound_tracker.is_same_execution(query_tracker))
285    }
286}
287
288#[derive(Clone)]
289pub struct SessionCtxDesc {
290    // maybe we need some info
291}
292
293pub struct SessionCtxFactory {
294    pub is_test: bool,
295    pub target_partitions: usize,
296}
297
298pub const DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES: usize = 64 * 1024 * 1024;
299
300impl Default for SessionCtxFactory {
301    fn default() -> Self {
302        Self::new(false)
303    }
304}
305
306impl SessionCtxFactory {
307    pub fn new(is_test: bool) -> Self {
308        Self {
309            is_test,
310            target_partitions: 0,
311        }
312    }
313
314    pub fn with_target_partitions(mut self, target_partitions: usize) -> Self {
315        self.target_partitions = target_partitions;
316        self
317    }
318
319    pub async fn create_session_ctx(&self, context: &Context) -> QueryResult<SessionCtx> {
320        self.create_session_ctx_inner(
321            context,
322            None,
323            legacy_json_source_from_input(&context.input),
324            None,
325            None,
326            DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
327        )
328        .await
329    }
330
331    pub async fn create_session_ctx_with_tracker_and_memory_limit(
332        &self,
333        context: &Context,
334        query_tracker: QueryExecutionTracker,
335        memory_limit_bytes: usize,
336    ) -> QueryResult<SessionCtx> {
337        self.create_session_ctx_inner(
338            context,
339            None,
340            legacy_json_source_from_input(&context.input),
341            Some(query_tracker),
342            None,
343            memory_limit_bytes,
344        )
345        .await
346    }
347
348    pub async fn create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
349        &self,
350        context: &Context,
351        snapshot: Arc<SelectObjectSnapshot>,
352        query_tracker: QueryExecutionTracker,
353        memory_limit_bytes: usize,
354    ) -> QueryResult<SessionCtx> {
355        self.create_session_ctx_inner(
356            context,
357            Some(snapshot),
358            legacy_json_source_from_input(&context.input),
359            Some(query_tracker),
360            None,
361            memory_limit_bytes,
362        )
363        .await
364    }
365
366    pub async fn create_session_ctx_for_query_with_source_and_tracker_and_memory_limit(
367        &self,
368        query: &Query,
369        source: JsonSource,
370        query_tracker: QueryExecutionTracker,
371        memory_limit_bytes: usize,
372    ) -> QueryResult<SessionCtx> {
373        self.create_session_ctx_inner(
374            query.context(),
375            query.snapshot().cloned(),
376            source,
377            Some(query_tracker),
378            Some(Arc::clone(query.input_metrics())),
379            memory_limit_bytes,
380        )
381        .await
382    }
383
384    pub async fn create_session_ctx_for_query_with_tracker_and_memory_limit(
385        &self,
386        query: &Query,
387        query_tracker: QueryExecutionTracker,
388        memory_limit_bytes: usize,
389    ) -> QueryResult<SessionCtx> {
390        self.create_session_ctx_inner(
391            query.context(),
392            query.snapshot().cloned(),
393            legacy_json_source_from_input(&query.context().input),
394            Some(query_tracker),
395            Some(Arc::clone(query.input_metrics())),
396            memory_limit_bytes,
397        )
398        .await
399    }
400
401    async fn create_session_ctx_inner(
402        &self,
403        context: &Context,
404        snapshot: Option<Arc<SelectObjectSnapshot>>,
405        source: JsonSource,
406        query_tracker: Option<QueryExecutionTracker>,
407        input_metrics: Option<Arc<SelectInputMetrics>>,
408        memory_limit_bytes: usize,
409    ) -> QueryResult<SessionCtx> {
410        let df_session_ctx = self
411            .build_df_session_context(context, snapshot, source, query_tracker.clone(), input_metrics, memory_limit_bytes)
412            .await?;
413
414        Ok(SessionCtx {
415            _desc: Arc::new(SessionCtxDesc {}),
416            inner: df_session_ctx.state(),
417            query_tracker,
418        })
419    }
420
421    async fn build_df_session_context(
422        &self,
423        context: &Context,
424        snapshot: Option<Arc<SelectObjectSnapshot>>,
425        source: JsonSource,
426        query_tracker: Option<QueryExecutionTracker>,
427        input_metrics: Option<Arc<SelectInputMetrics>>,
428        memory_limit_bytes: usize,
429    ) -> QueryResult<SessionContext> {
430        let path = format!("s3://{}", context.input.bucket);
431        let store_url = url::Url::parse(&path).unwrap();
432        let memory_limit_bytes = if memory_limit_bytes == 0 {
433            DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES
434        } else {
435            memory_limit_bytes
436        };
437        let rt = RuntimeEnvBuilder::new().with_memory_limit(memory_limit_bytes, 1.0).build()?;
438        let config = SessionConfig::new().with_target_partitions(self.target_partitions);
439        let custom_two_byte_record_delimiter = context
440            .input
441            .request
442            .input_serialization
443            .csv
444            .as_ref()
445            .and_then(|csv| csv.record_delimiter.as_deref())
446            .is_some_and(|delimiter| delimiter.len() == 2 && delimiter.as_bytes() != b"\r\n");
447        let scan_range_requires_single_file_scan =
448            context.input.request.scan_range.is_some() && context.input.request.input_serialization.parquet.is_none();
449        let json_document_requires_single_file_scan = is_json_document_input(&context.input);
450        let compressed_input_requires_single_file_scan = context
451            .input
452            .request
453            .input_serialization
454            .compression_type
455            .as_ref()
456            .is_some_and(|compression| compression.as_str() != CompressionType::NONE);
457        let metered_input_requires_single_file_scan =
458            input_metrics.is_some() && context.input.request.input_serialization.parquet.is_none();
459        let normalized_csv_requires_single_file_scan =
460            context.input.request.input_serialization.csv.as_ref().is_some_and(|csv| {
461                crate::csv_input_requires_normalization(csv.quote_character.as_deref(), csv.quote_escape_character.as_deref())
462            });
463        let config = if normalized_csv_requires_single_file_scan
464            || custom_two_byte_record_delimiter
465            || scan_range_requires_single_file_scan
466            || json_document_requires_single_file_scan
467            || compressed_input_requires_single_file_scan
468            || metered_input_requires_single_file_scan
469        {
470            config.with_repartition_file_scans(false)
471        } else {
472            config
473        };
474        let memory_pool = Arc::clone(&rt.memory_pool);
475        let df_session_state = SessionStateBuilder::new()
476            .with_config(config)
477            .with_runtime_env(Arc::new(rt))
478            .with_default_features();
479
480        let df_session_state = if self.is_test {
481            let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
482
483            // Choose test data format based on what the request serialization specifies.
484            let data_bytes: Vec<u8> = if context.input.request.input_serialization.parquet.is_some() {
485                test_parquet_bytes()?
486            } else if context.input.request.input_serialization.json.is_some() {
487                // NDJSON: one JSON object per line — usable for both LINES and DOCUMENT
488                // requests (DOCUMENT inputs are converted to NDJSON by EcObjectStore, but
489                // in test mode we bypass EcObjectStore, so we put NDJSON here directly).
490                b"{\"id\":1,\"name\":\"Alice\",\"age\":25,\"department\":\"HR\",\"salary\":5000}\n\
491                   {\"id\":2,\"name\":\"Bob\",\"age\":30,\"department\":\"IT\",\"salary\":6000}\n\
492                   {\"id\":3,\"name\":\"Charlie\",\"age\":35,\"department\":\"Finance\",\"salary\":7000}\n\
493                   {\"id\":4,\"name\":\"Diana\",\"age\":22,\"department\":\"Marketing\",\"salary\":4500}\n\
494                   {\"id\":5,\"name\":\"Eve\",\"age\":28,\"department\":\"IT\",\"salary\":5500}\n\
495                   {\"id\":6,\"name\":\"Frank\",\"age\":40,\"department\":\"Finance\",\"salary\":8000}\n\
496                   {\"id\":7,\"name\":\"Grace\",\"age\":26,\"department\":\"HR\",\"salary\":5200}\n\
497                   {\"id\":8,\"name\":\"Henry\",\"age\":32,\"department\":\"IT\",\"salary\":6200}\n\
498                   {\"id\":9,\"name\":\"Ivy\",\"age\":24,\"department\":\"Marketing\",\"salary\":4800}\n\
499                   {\"id\":10,\"name\":\"Jack\",\"age\":38,\"department\":\"Finance\",\"salary\":7500}\n"
500                    .to_vec()
501            } else {
502                b"id,name,age,department,salary\n\
503                  1,Alice,25,HR,05000\n\
504                  2,Bob,30,IT,6000\n\
505                  3,Charlie,35,Finance,7000\n\
506                  4,Diana,22,Marketing,4500\n\
507                  5,Eve,28,IT,5500\n\
508                  6,Frank,40,Finance,8000\n\
509                  7,Grace,26,HR,5200\n\
510                  8,Henry,32,IT,6200\n\
511                  9,Ivy,24,Marketing,4800\n\
512                  10,Jack,38,Finance,7500"
513                    .to_vec()
514            };
515
516            let path = Path::from(context.input.key.clone());
517            store.put(&path, data_bytes.into()).await.map_err(|e| {
518                error!("put data into memory failed: {}", e.to_string());
519                QueryError::from(DataFusionError::from(e))
520            })?;
521
522            df_session_state.with_object_store(&store_url, store).build()
523        } else {
524            let input_metrics = input_metrics.unwrap_or_else(|| Arc::new(SelectInputMetrics::default()));
525            let store: EcObjectStore = match query_tracker {
526                Some(query_tracker) => EcObjectStore::new_with_query_tracker_and_source(
527                    context.input.clone(),
528                    memory_pool,
529                    query_tracker,
530                    input_metrics,
531                    snapshot,
532                    source,
533                ),
534                None => EcObjectStore::new_with_memory_pool_and_source(
535                    context.input.clone(),
536                    memory_pool,
537                    input_metrics,
538                    snapshot,
539                    source,
540                ),
541            }
542            .map_err(|err| QueryError::Datafusion {
543                source: Box::new(DataFusionError::External(Box::new(err))),
544            })?;
545            df_session_state.with_object_store(&store_url, Arc::new(store)).build()
546        };
547
548        let df_session_ctx = SessionContext::new_with_state(df_session_state);
549
550        Ok(df_session_ctx)
551    }
552}
553
554fn test_parquet_bytes() -> QueryResult<Vec<u8>> {
555    let schema = Arc::new(Schema::new(vec![
556        Field::new("id", DataType::Int32, false),
557        Field::new("name", DataType::Utf8, false),
558        Field::new("age", DataType::Int32, false),
559        Field::new("department", DataType::Utf8, false),
560        Field::new("salary", DataType::Int32, false),
561    ]));
562    let first_batch =
563        test_parquet_batch(Arc::clone(&schema), &[1, 2], &["Alice", "Bob"], &[25, 30], &["HR", "IT"], &[5000, 6000])?;
564    let second_batch = test_parquet_batch(
565        Arc::clone(&schema),
566        &[3, 4, 5],
567        &["Charlie", "Diana", "Eve"],
568        &[35, 22, 28],
569        &["Finance", "Marketing", "IT"],
570        &[7000, 4500, 5500],
571    )?;
572
573    let mut bytes = Vec::new();
574    {
575        let mut writer = ArrowWriter::try_new(&mut bytes, schema, None).map_err(DataFusionError::from)?;
576        writer.write(&first_batch).map_err(DataFusionError::from)?;
577        writer.flush().map_err(DataFusionError::from)?;
578        writer.write(&second_batch).map_err(DataFusionError::from)?;
579        writer.close().map_err(DataFusionError::from)?;
580    }
581    Ok(bytes)
582}
583
584fn test_parquet_batch(
585    schema: Arc<Schema>,
586    ids: &[i32],
587    names: &[&str],
588    ages: &[i32],
589    departments: &[&str],
590    salaries: &[i32],
591) -> QueryResult<RecordBatch> {
592    RecordBatch::try_new(
593        schema,
594        vec![
595            Arc::new(Int32Array::from(ids.to_vec())),
596            Arc::new(StringArray::from(names.to_vec())),
597            Arc::new(Int32Array::from(ages.to_vec())),
598            Arc::new(StringArray::from(departments.to_vec())),
599            Arc::new(Int32Array::from(salaries.to_vec())),
600        ],
601    )
602    .map_err(DataFusionError::from)
603    .map_err(QueryError::from)
604}
605
606#[cfg(test)]
607mod tests {
608    use super::*;
609    use crate::storage_api::SelectPutObjReader;
610    use crate::storage_api::object_store::ObjectIO as _;
611    use datafusion::{
612        datasource::{
613            file_format::{csv::CsvFormat, json::JsonFormat},
614            listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
615        },
616        execution::memory_pool::MemoryLimit,
617    };
618    use http::HeaderMap;
619    use s3s::dto::{
620        CSVInput, CSVOutput, ExpressionType, InputSerialization, JSONInput, JSONType, OutputSerialization, ParquetInput,
621        ScanRange, SelectObjectContentInput, SelectObjectContentRequest,
622    };
623    use std::io::Write as _;
624
625    fn test_context() -> Context {
626        Context {
627            input: Arc::new(SelectObjectContentInput {
628                bucket: "test-bucket".to_string(),
629                expected_bucket_owner: None,
630                key: "test.csv".to_string(),
631                sse_customer_algorithm: None,
632                sse_customer_key: None,
633                sse_customer_key_md5: None,
634                request: SelectObjectContentRequest {
635                    expression: "SELECT * FROM S3Object".to_string(),
636                    expression_type: ExpressionType::from_static("SQL"),
637                    input_serialization: InputSerialization {
638                        csv: Some(CSVInput::default()),
639                        ..Default::default()
640                    },
641                    output_serialization: OutputSerialization {
642                        csv: Some(CSVOutput::default()),
643                        ..Default::default()
644                    },
645                    request_progress: None,
646                    scan_range: None,
647                },
648            }),
649        }
650    }
651
652    async fn prepare_test_snapshot(context: &Context) -> Arc<SelectObjectSnapshot> {
653        let env = crate::storage_api::select_test_ecstore_env().await;
654        Arc::new(
655            env.ecstore
656                .prepare_select_object_snapshot(&context.input.bucket, &context.input.key, &HeaderMap::new(), &Default::default())
657                .await
658                .expect("prepare SelectObjectContent snapshot"),
659        )
660    }
661
662    async fn test_query_tracker() -> QueryExecutionTracker {
663        let permit = Arc::new(tokio::sync::Semaphore::new(1))
664            .acquire_owned()
665            .await
666            .expect("query permit should be available");
667        QueryExecutionTracker::new(
668            &QueryExecutionOwner::new(),
669            Arc::new(permit),
670            Instant::now() + std::time::Duration::from_secs(300),
671            300,
672        )
673    }
674
675    async fn assert_legacy_json_column(bucket: &str, expression: &str, document: &[u8], column_name: &str, expected: &[&str]) {
676        const OBJECT: &str = "input.json";
677
678        let env = crate::storage_api::select_test_ecstore_env().await;
679        let mut context = test_context();
680        {
681            let input = Arc::make_mut(&mut context.input);
682            input.bucket = bucket.to_string();
683            input.key = OBJECT.to_string();
684            input.request.expression = expression.to_string();
685            input.request.input_serialization.csv = None;
686            input.request.input_serialization.json = Some(JSONInput {
687                type_: Some(JSONType::from_static(JSONType::DOCUMENT)),
688            });
689        }
690        env.make_bucket(bucket, false).await;
691        env.put_object_bytes(bucket, OBJECT, document.to_vec()).await;
692        let factory = SessionCtxFactory::new(false);
693        let lazy_session = factory
694            .create_session_ctx(&context)
695            .await
696            .expect("legacy lazy session should preserve the JSON source");
697        let tracked_lazy_session = factory
698            .create_session_ctx_with_tracker_and_memory_limit(
699                &context,
700                test_query_tracker().await,
701                DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
702            )
703            .await
704            .expect("legacy tracked lazy session should preserve the JSON source");
705        let snapshot = prepare_test_snapshot(&context).await;
706        let snapshot_session = factory
707            .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
708                &context,
709                snapshot,
710                test_query_tracker().await,
711                DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
712            )
713            .await
714            .expect("legacy snapshot session should preserve the JSON source");
715
716        for (kind, session) in [
717            ("lazy", lazy_session),
718            ("tracked lazy", tracked_lazy_session),
719            ("tracked snapshot", snapshot_session),
720        ] {
721            let table_path = ListingTableUrl::parse(format!("s3://{bucket}/{OBJECT}")).expect("parse JSON table URL");
722            let listing_options = ListingOptions::new(Arc::new(JsonFormat::default())).with_file_extension(".json");
723            let schema = listing_options
724                .infer_schema(session.inner(), &table_path)
725                .await
726                .expect("infer expanded JSON schema");
727            let table = ListingTable::try_new(
728                ListingTableConfig::new(table_path)
729                    .with_listing_options(listing_options)
730                    .with_schema(schema),
731            )
732            .expect("build expanded JSON table");
733            let query_context = SessionContext::new_with_state(session.inner().clone());
734            query_context
735                .register_table("legacy_input", Arc::new(table))
736                .expect("register expanded JSON table");
737            let batches = query_context
738                .sql(&format!("SELECT {column_name} FROM legacy_input"))
739                .await
740                .expect("plan expanded JSON query")
741                .collect()
742                .await
743                .expect("execute expanded JSON query");
744            let mut values = Vec::new();
745            for batch in batches {
746                let column = batch
747                    .column(0)
748                    .as_any()
749                    .downcast_ref::<StringArray>()
750                    .expect("expanded column should be Utf8");
751                for row in 0..batch.num_rows() {
752                    values.push(column.value(row).to_string());
753                }
754            }
755            assert_eq!(values, expected, "{kind} legacy constructor");
756        }
757    }
758
759    #[test]
760    fn session_factory_fields_remain_source_compatible() {
761        let factory = SessionCtxFactory {
762            is_test: true,
763            target_partitions: 0,
764        };
765
766        assert!(factory.is_test);
767        assert_eq!(factory.target_partitions, 0);
768    }
769
770    #[tokio::test]
771    async fn session_factory_applies_target_partitions() {
772        let factory = SessionCtxFactory::new(true).with_target_partitions(3);
773        let session = factory
774            .create_session_ctx(&test_context())
775            .await
776            .expect("session should be created with configured target partitions");
777
778        assert_eq!(session.inner().config().target_partitions(), 3);
779        assert!(session.inner().config().options().optimizer.repartition_file_scans);
780    }
781
782    #[tokio::test]
783    async fn metered_csv_and_json_inputs_disable_file_repartitioning() {
784        let factory = SessionCtxFactory::new(true).with_target_partitions(3);
785        let csv_context = test_context();
786        let mut json_context = test_context();
787        let json_request = &mut Arc::make_mut(&mut json_context.input).request;
788        json_request.input_serialization.csv = None;
789        json_request.input_serialization.json = Some(JSONInput::default());
790
791        for context in [&csv_context, &json_context] {
792            let session = factory
793                .create_session_ctx_inner(
794                    context,
795                    None,
796                    JsonSource::default(),
797                    None,
798                    Some(Arc::new(SelectInputMetrics::default())),
799                    DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
800                )
801                .await
802                .expect("metered session should be created");
803
804            assert!(!session.inner().config().options().optimizer.repartition_file_scans);
805        }
806    }
807
808    #[tokio::test]
809    async fn parquet_scan_range_keeps_file_repartitioning() {
810        let mut context = test_context();
811        let request = &mut Arc::make_mut(&mut context.input).request;
812        request.scan_range = Some(ScanRange {
813            start: Some(0),
814            end: Some(0),
815        });
816        request.input_serialization.csv = None;
817        request.input_serialization.parquet = Some(ParquetInput {});
818
819        let session = SessionCtxFactory::new(true)
820            .with_target_partitions(2)
821            .create_session_ctx(&context)
822            .await
823            .expect("Parquet ScanRange session should be created");
824
825        assert!(session.inner().config().options().optimizer.repartition_file_scans);
826    }
827
828    #[tokio::test]
829    async fn json_lines_scan_range_disables_file_repartitioning() {
830        let mut context = test_context();
831        let request = &mut Arc::make_mut(&mut context.input).request;
832        request.scan_range = Some(ScanRange {
833            start: Some(0),
834            end: Some(0),
835        });
836        request.input_serialization.csv = None;
837        request.input_serialization.json = Some(JSONInput::default());
838
839        let session = SessionCtxFactory::new(true)
840            .with_target_partitions(2)
841            .create_session_ctx(&context)
842            .await
843            .expect("JSON LINES ScanRange session should be created");
844
845        assert!(!session.inner().config().options().optimizer.repartition_file_scans);
846    }
847
848    #[tokio::test]
849    async fn json_lines_without_scan_range_keeps_file_repartitioning() {
850        let mut context = test_context();
851        let request = &mut Arc::make_mut(&mut context.input).request;
852        request.input_serialization.csv = None;
853        request.input_serialization.json = Some(JSONInput::default());
854
855        let session = SessionCtxFactory::new(true)
856            .with_target_partitions(2)
857            .create_session_ctx(&context)
858            .await
859            .expect("JSON LINES session should be created");
860
861        assert!(session.inner().config().options().optimizer.repartition_file_scans);
862    }
863
864    #[tokio::test]
865    async fn compressed_input_disables_file_repartitioning_without_metrics() {
866        let mut context = test_context();
867        Arc::make_mut(&mut context.input).request.input_serialization.compression_type =
868            Some(CompressionType::from_static(CompressionType::GZIP));
869
870        let session = SessionCtxFactory::new(true)
871            .with_target_partitions(2)
872            .create_session_ctx(&context)
873            .await
874            .expect("compressed session should be created");
875
876        assert!(!session.inner().config().options().optimizer.repartition_file_scans);
877    }
878
879    #[tokio::test]
880    async fn json_document_disables_file_repartitioning() {
881        let mut context = test_context();
882        let request = &mut Arc::make_mut(&mut context.input).request;
883        request.input_serialization.csv = None;
884        request.input_serialization.json = Some(JSONInput {
885            type_: Some(JSONType::from_static(JSONType::DOCUMENT)),
886        });
887
888        let session = SessionCtxFactory::new(true)
889            .with_target_partitions(2)
890            .create_session_ctx(&context)
891            .await
892            .expect("JSON DOCUMENT session should be created");
893
894        assert!(!session.inner().config().options().optimizer.repartition_file_scans);
895    }
896
897    #[tokio::test]
898    async fn csv_scan_range_disables_file_repartitioning() {
899        let mut context = test_context();
900        Arc::make_mut(&mut context.input).request.scan_range = Some(ScanRange {
901            start: Some(0),
902            end: Some(0),
903        });
904
905        let session = SessionCtxFactory::new(true)
906            .with_target_partitions(2)
907            .create_session_ctx(&context)
908            .await
909            .expect("CSV ScanRange session should be created");
910
911        assert!(!session.inner().config().options().optimizer.repartition_file_scans);
912    }
913
914    #[tokio::test]
915    async fn unicode_csv_quotes_disable_file_scan_repartition() {
916        let mut context = test_context();
917        Arc::get_mut(&mut context.input)
918            .expect("unique context")
919            .request
920            .input_serialization
921            .csv
922            .as_mut()
923            .expect("CSV input")
924            .quote_character = Some("ع".to_owned());
925        let session = SessionCtxFactory::new(true)
926            .with_target_partitions(4)
927            .create_session_ctx(&context)
928            .await
929            .expect("Unicode CSV session");
930        assert!(!session.inner().config().options().optimizer.repartition_file_scans);
931    }
932
933    #[tokio::test]
934    async fn two_byte_csv_record_delimiter_disables_file_scan_repartition() {
935        let mut context = test_context();
936        Arc::get_mut(&mut context.input)
937            .expect("test context input should be uniquely owned")
938            .request
939            .input_serialization
940            .csv
941            .as_mut()
942            .expect("test context should use CSV")
943            .record_delimiter = Some("^Y".to_string());
944        let session = SessionCtxFactory::new(true)
945            .with_target_partitions(4)
946            .create_session_ctx(&context)
947            .await
948            .expect("session should be created");
949
950        assert!(!session.inner().config().options().optimizer.repartition_file_scans);
951    }
952
953    #[tokio::test]
954    async fn crlf_record_delimiter_retains_file_scan_repartition() {
955        let mut context = test_context();
956        Arc::get_mut(&mut context.input)
957            .expect("test context input should be uniquely owned")
958            .request
959            .input_serialization
960            .csv
961            .as_mut()
962            .expect("test context should use CSV")
963            .record_delimiter = Some("\r\n".to_string());
964        let session = SessionCtxFactory::new(true)
965            .with_target_partitions(4)
966            .create_session_ctx(&context)
967            .await
968            .expect("session should be created");
969
970        assert_eq!(
971            session.inner().config().options().optimizer.repartition_file_scans,
972            SessionConfig::new().options().optimizer.repartition_file_scans
973        );
974    }
975
976    #[tokio::test]
977    async fn session_factory_zero_target_partitions_uses_datafusion_default() {
978        let factory = SessionCtxFactory::new(true);
979        let session = factory
980            .create_session_ctx(&test_context())
981            .await
982            .expect("session should be created with default target partitions");
983
984        assert_eq!(session.inner().config().target_partitions(), SessionConfig::new().target_partitions());
985    }
986
987    #[tokio::test]
988    async fn session_factory_applies_memory_limit() {
989        let factory = SessionCtxFactory::new(true);
990        let session = factory
991            .create_session_ctx_inner(&test_context(), None, JsonSource::default(), None, None, 1024)
992            .await
993            .expect("session should be created with a bounded memory pool");
994
995        assert!(matches!(
996            session.inner().runtime_env().memory_pool.memory_limit(),
997            MemoryLimit::Finite(1024)
998        ));
999    }
1000
1001    #[tokio::test]
1002    #[serial_test::serial]
1003    async fn production_session_preserves_lazy_snapshot_entry() {
1004        let _env = crate::storage_api::select_test_ecstore_env().await;
1005        let session = SessionCtxFactory::new(false)
1006            .create_session_ctx(&test_context())
1007            .await
1008            .expect("legacy production session should install a lazy object store");
1009
1010        assert_eq!(session.inner().config().target_partitions(), SessionConfig::new().target_partitions());
1011    }
1012
1013    #[tokio::test]
1014    #[serial_test::serial]
1015    async fn legacy_tracked_production_session_preserves_lazy_snapshot_entry() {
1016        let _env = crate::storage_api::select_test_ecstore_env().await;
1017        let permit = Arc::new(tokio::sync::Semaphore::new(1))
1018            .acquire_owned()
1019            .await
1020            .expect("query permit should be available");
1021        let tracker = QueryExecutionTracker::new(
1022            &QueryExecutionOwner::new(),
1023            Arc::new(permit),
1024            Instant::now() + std::time::Duration::from_secs(300),
1025            300,
1026        );
1027        let session = SessionCtxFactory::new(false)
1028            .create_session_ctx_with_tracker_and_memory_limit(
1029                &test_context(),
1030                tracker.clone(),
1031                DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1032            )
1033            .await
1034            .expect("legacy tracked session should install a lazy object store");
1035
1036        assert!(session.is_bound_to(&tracker));
1037    }
1038
1039    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1040    #[serial_test::serial]
1041    async fn legacy_session_factory_preserves_single_key_json_source() {
1042        assert_legacy_json_column(
1043            "s3select-legacy-session-json-source",
1044            "SELECT e.name FROM S3Object.employees AS e",
1045            br#"{"employees":[{"name":"Alice"}]}"#,
1046            "name",
1047            &["Alice"],
1048        )
1049        .await;
1050    }
1051
1052    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1053    #[serial_test::serial]
1054    async fn legacy_session_factory_preserves_implicit_root_alias() {
1055        assert_legacy_json_column(
1056            "s3select-legacy-session-root-alias",
1057            "SELECT S3Object FROM S3Object",
1058            br#"["one","two"]"#,
1059            "s3object",
1060            &["one", "two"],
1061        )
1062        .await;
1063    }
1064
1065    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1066    #[serial_test::serial]
1067    async fn legacy_session_factory_preserves_quoted_root_alias() {
1068        assert_legacy_json_column(
1069            "s3select-legacy-session-quoted-root-alias",
1070            "SELECT \"V\" FROM S3Object AS \"V\"",
1071            br#"["one","two"]"#,
1072            "\"V\"",
1073            &["one", "two"],
1074        )
1075        .await;
1076    }
1077
1078    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1079    #[serial_test::serial]
1080    async fn session_factory_propagates_query_guard_to_ec_store() {
1081        let env = crate::storage_api::select_test_ecstore_env().await;
1082        let mut context = test_context();
1083        Arc::make_mut(&mut context.input).bucket = "s3select-query-guard-snapshot".to_string();
1084        env.make_bucket(&context.input.bucket, false).await;
1085        let mut reader = SelectPutObjReader::from_vec(b"id,name\n1,Alice\n".to_vec());
1086        env.ecstore
1087            .put_object(&context.input.bucket, &context.input.key, &mut reader, &Default::default())
1088            .await
1089            .expect("put query guard fixture");
1090        let snapshot = prepare_test_snapshot(&context).await;
1091
1092        let admission = Arc::new(tokio::sync::Semaphore::new(1));
1093        let permit = Arc::clone(&admission)
1094            .acquire_owned()
1095            .await
1096            .expect("query permit should be available");
1097        let query_guard = Arc::new(permit);
1098        let query_tracker = QueryExecutionTracker::new(
1099            &QueryExecutionOwner::new(),
1100            Arc::clone(&query_guard),
1101            Instant::now() + std::time::Duration::from_secs(300),
1102            300,
1103        );
1104        let session = SessionCtxFactory::new(false)
1105            .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
1106                &context,
1107                snapshot,
1108                query_tracker,
1109                DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1110            )
1111            .await
1112            .expect("production session should be created with the query guard");
1113
1114        assert!(Arc::strong_count(&query_guard) > 1);
1115        drop(session);
1116        assert_eq!(Arc::strong_count(&query_guard), 1);
1117    }
1118
1119    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1120    #[serial_test::serial]
1121    async fn session_factory_preserves_snapshot_binding_error_source() {
1122        let env = crate::storage_api::select_test_ecstore_env().await;
1123        let mut source_context = test_context();
1124        Arc::make_mut(&mut source_context.input).bucket = "s3select-session-snapshot-identity".to_string();
1125        Arc::make_mut(&mut source_context.input).key = "source.csv".to_string();
1126        env.make_bucket(&source_context.input.bucket, false).await;
1127        let mut reader = SelectPutObjReader::from_vec(b"source-marker\n".to_vec());
1128        env.ecstore
1129            .put_object(&source_context.input.bucket, &source_context.input.key, &mut reader, &Default::default())
1130            .await
1131            .expect("put snapshot identity fixture");
1132        let snapshot = prepare_test_snapshot(&source_context).await;
1133
1134        let mut target_context = source_context.clone();
1135        Arc::make_mut(&mut target_context.input).key = "different.csv".to_string();
1136        let permit = Arc::new(tokio::sync::Semaphore::new(1))
1137            .acquire_owned()
1138            .await
1139            .expect("query permit should be available");
1140        let tracker = QueryExecutionTracker::new(
1141            &QueryExecutionOwner::new(),
1142            Arc::new(permit),
1143            Instant::now() + std::time::Duration::from_secs(300),
1144            300,
1145        );
1146
1147        let error = match SessionCtxFactory::new(false)
1148            .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
1149                &target_context,
1150                snapshot,
1151                tracker,
1152                DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1153            )
1154            .await
1155        {
1156            Ok(_) => panic!("a session must reject a snapshot for a different object"),
1157            Err(error) => error,
1158        };
1159
1160        assert!(error.is_snapshot_consistency_error());
1161        assert!(error.to_string().contains("snapshot consistency failure"));
1162    }
1163
1164    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1165    #[serial_test::serial]
1166    async fn scan_range_is_preserved_across_large_csv_partition_boundary() {
1167        const ROW_COUNT: usize = 200_000;
1168        const ROW_WIDTH: usize = 7;
1169        const SELECTED_ROW: usize = 100_000;
1170
1171        let env = crate::storage_api::select_test_ecstore_env().await;
1172        let mut data = Vec::with_capacity(ROW_COUNT * ROW_WIDTH);
1173        for row in 0..ROW_COUNT {
1174            writeln!(&mut data, "{row:06}").expect("write fixed-width CSV test row");
1175        }
1176        assert!(data.len() > 1024 * 1024);
1177
1178        let mut context = test_context();
1179        Arc::make_mut(&mut context.input).bucket = "s3select-scan-range-partition-snapshot".to_string();
1180        let selected_start = i64::try_from(SELECTED_ROW * ROW_WIDTH).expect("selected row offset should fit in i64");
1181        Arc::make_mut(&mut context.input).request.scan_range = Some(ScanRange {
1182            start: Some(selected_start),
1183            end: Some(selected_start),
1184        });
1185        env.make_bucket(&context.input.bucket, false).await;
1186        let mut reader = SelectPutObjReader::from_vec(data);
1187        env.ecstore
1188            .put_object(&context.input.bucket, &context.input.key, &mut reader, &Default::default())
1189            .await
1190            .expect("put large ScanRange CSV fixture");
1191        let snapshot = prepare_test_snapshot(&context).await;
1192
1193        let admission = Arc::new(tokio::sync::Semaphore::new(1));
1194        let permit = Arc::clone(&admission)
1195            .acquire_owned()
1196            .await
1197            .expect("query permit should be available");
1198        let query_tracker = QueryExecutionTracker::new(
1199            &QueryExecutionOwner::new(),
1200            Arc::new(permit),
1201            Instant::now() + std::time::Duration::from_secs(300),
1202            300,
1203        );
1204        let session = SessionCtxFactory::new(false)
1205            .with_target_partitions(2)
1206            .create_session_ctx_with_snapshot_and_tracker_and_memory_limit(
1207                &context,
1208                snapshot,
1209                query_tracker,
1210                DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
1211            )
1212            .await
1213            .expect("create production ScanRange session");
1214        assert!(!session.inner().config().options().optimizer.repartition_file_scans);
1215
1216        let table_path = ListingTableUrl::parse(format!("s3://{}/{}", context.input.bucket, context.input.key))
1217            .expect("parse ScanRange table URL");
1218        let listing_options =
1219            ListingOptions::new(Arc::new(CsvFormat::default().with_schema_infer_max_rec(0).with_has_header(false)))
1220                .with_file_extension(".csv");
1221        let schema = listing_options
1222            .infer_schema(session.inner(), &table_path)
1223            .await
1224            .expect("infer ScanRange CSV schema");
1225        let table = ListingTable::try_new(
1226            ListingTableConfig::new(table_path)
1227                .with_listing_options(listing_options)
1228                .with_schema(schema),
1229        )
1230        .expect("build ScanRange listing table");
1231        let query_context = SessionContext::new_with_state(session.inner().clone());
1232        query_context
1233            .register_table("scan_input", Arc::new(table))
1234            .expect("register ScanRange listing table");
1235
1236        let batches = query_context
1237            .sql("SELECT * FROM scan_input")
1238            .await
1239            .expect("plan ScanRange CSV query")
1240            .collect()
1241            .await
1242            .expect("execute ScanRange CSV query");
1243        let mut values = Vec::new();
1244        for batch in batches {
1245            let column = batch
1246                .column(0)
1247                .as_any()
1248                .downcast_ref::<StringArray>()
1249                .expect("ScanRange CSV column should be Utf8");
1250            for row in 0..batch.num_rows() {
1251                values.push(column.value(row).to_string());
1252            }
1253        }
1254
1255        assert_eq!(values.len(), 1);
1256        assert_eq!(values, [format!("{SELECTED_ROW:06}")]);
1257    }
1258
1259    #[tokio::test]
1260    async fn elapsed_deadline_does_not_yield_query_guard_before_timer_poll() {
1261        let admission = Arc::new(tokio::sync::Semaphore::new(1));
1262        let permit = Arc::clone(&admission)
1263            .acquire_owned()
1264            .await
1265            .expect("query permit should be available");
1266        let query_tracker = QueryExecutionTracker::new(&QueryExecutionOwner::new(), Arc::new(permit), Instant::now(), 0);
1267
1268        assert!(query_tracker.query_guard().is_none());
1269    }
1270
1271    #[tokio::test]
1272    async fn session_factory_default_uses_bounded_memory() {
1273        let factory = SessionCtxFactory::default();
1274        let session = SessionCtxFactory::new(true)
1275            .create_session_ctx(&test_context())
1276            .await
1277            .expect("default session should be created with a bounded memory pool");
1278
1279        assert!(!factory.is_test);
1280        assert_eq!(factory.target_partitions, 0);
1281        assert!(matches!(
1282            session.inner().runtime_env().memory_pool.memory_limit(),
1283            MemoryLimit::Finite(DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES)
1284        ));
1285    }
1286}