rustfs_s3select_api/query/
execution.rs1use std::fmt::Display;
16use std::pin::Pin;
17use std::sync::Arc;
18use std::task::{Context, Poll};
19use std::time::{Duration, Instant};
20
21use async_trait::async_trait;
22use datafusion::arrow::datatypes::{Schema, SchemaRef};
23use datafusion::arrow::record_batch::RecordBatch;
24use datafusion::physical_plan::SendableRecordBatchStream;
25use futures::{Stream, StreamExt, TryStreamExt};
26use parking_lot::RwLock;
27use tracing::debug;
28
29use crate::{QueryError, QueryResult};
30
31use super::Query;
32use super::ast::ExtStatement;
33use super::logical_planner::Plan;
34use super::session::{QueryExecutionTracker, SessionCtx};
35
36pub struct PhaseTimer {
37 phase_name: &'static str,
38 start_time: Instant,
39}
40
41impl PhaseTimer {
42 pub fn new(phase_name: &'static str) -> Self {
43 Self {
44 phase_name,
45 start_time: Instant::now(),
46 }
47 }
48}
49
50impl Drop for PhaseTimer {
51 fn drop(&mut self) {
52 let duration = self.start_time.elapsed();
53
54 if !std::thread::panicking() {
55 metrics::histogram!(
56 "rustfs_s3select_phase_duration_seconds",
57 "phase" => self.phase_name
58 )
59 .record(duration.as_secs_f64());
60 }
61
62 debug!("Phase '{}' took {:?}", self.phase_name, duration);
63 }
64}
65
66pub type QueryExecutionRef = Arc<dyn QueryExecution>;
67
68#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub enum QueryType {
70 Batch,
71 Stream,
72}
73
74impl Display for QueryType {
75 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
76 match self {
77 Self::Batch => write!(f, "batch"),
78 Self::Stream => write!(f, "stream"),
79 }
80 }
81}
82
83#[async_trait]
84pub trait QueryExecution: Send + Sync {
85 fn query_type(&self) -> QueryType {
86 QueryType::Batch
87 }
88 async fn start(&self) -> QueryResult<Output>;
90 fn cancel(&self) -> QueryResult<()>;
92}
93
94pub enum Output {
95 StreamData(SendableRecordBatchStream),
96 Nil(()),
97}
98
99impl Output {
100 pub fn schema(&self) -> SchemaRef {
101 match self {
102 Self::StreamData(stream) => stream.schema(),
103 Self::Nil(_) => Arc::new(Schema::empty()),
104 }
105 }
106
107 pub async fn chunk_result(self) -> QueryResult<Vec<RecordBatch>> {
108 match self {
109 Self::Nil(_) => Ok(vec![]),
110 Self::StreamData(stream) => {
111 let schema = stream.schema();
112 let mut res: Vec<RecordBatch> = stream.try_collect::<Vec<RecordBatch>>().await?;
113 if res.is_empty() {
114 res.push(RecordBatch::new_empty(schema));
115 }
116 Ok(res)
117 }
118 }
119 }
120
121 pub fn into_record_batch_stream(self) -> QueryResult<SendableRecordBatchStream> {
122 match self {
123 Self::StreamData(stream) => Ok(stream),
124 Self::Nil(_) => Err(QueryError::NotImplemented {
125 err: "empty select output stream".to_string(),
126 }),
127 }
128 }
129
130 pub async fn num_rows(self) -> usize {
131 match self.chunk_result().await {
132 Ok(rb) => rb.iter().map(|e| e.num_rows()).sum(),
133 Err(_) => 0,
134 }
135 }
136
137 pub async fn affected_rows(self) -> i64 {
145 self.num_rows().await as i64
146 }
147}
148
149impl Stream for Output {
150 type Item = Result<RecordBatch, QueryError>;
151
152 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
153 let this = self.get_mut();
154 match this {
155 Output::StreamData(stream) => stream.poll_next_unpin(cx).map_err(|e| e.into()),
156 Output::Nil(_) => Poll::Ready(None),
157 }
158 }
159}
160
161#[async_trait]
162pub trait QueryExecutionFactory {
163 async fn create_query_execution(
164 &self,
165 plan: Plan,
166 query_state_machine: QueryStateMachineRef,
167 ) -> QueryResult<QueryExecutionRef>;
168}
169
170pub type QueryStateMachineRef = Arc<QueryStateMachine>;
171
172pub struct QueryStateMachine {
173 pub session: SessionCtx,
174 pub query: Query,
175
176 prepared_statement: Option<ExtStatement>,
177 query_tracker: Option<QueryExecutionTracker>,
178 state: RwLock<QueryState>,
179 start: Instant,
180}
181
182impl QueryStateMachine {
183 fn record_phase_timestamp(&self, phase: &'static str, event: &'static str) {
184 let elapsed = self.start.elapsed();
185 metrics::histogram!(
186 "rustfs_s3select_phase_timestamp_seconds",
187 "phase" => phase,
188 "event" => event
189 )
190 .record(elapsed.as_secs_f64());
191 }
192 #[must_use]
193 pub fn time_phase(&self, phase_name: &'static str) -> PhaseTimer {
194 PhaseTimer::new(phase_name)
195 }
196
197 pub fn begin(query: Query, session: SessionCtx) -> Self {
198 Self {
199 session,
200 query,
201 prepared_statement: None,
202 query_tracker: None,
203 state: RwLock::new(QueryState::ACCEPTING),
204 start: Instant::now(),
205 }
206 }
207
208 pub fn begin_tracked(query: Query, session: SessionCtx, query_tracker: QueryExecutionTracker) -> QueryResult<Self> {
209 if !session.is_bound_to(&query_tracker) {
210 return Err(QueryError::Cancel);
211 }
212 let mut state_machine = Self::begin(query, session);
213 state_machine.query_tracker = Some(query_tracker);
214 Ok(state_machine)
215 }
216
217 pub fn begin_tracked_prepared(
218 query: Query,
219 session: SessionCtx,
220 query_tracker: QueryExecutionTracker,
221 prepared_statement: ExtStatement,
222 ) -> QueryResult<Self> {
223 let mut state_machine = Self::begin_tracked(query, session, query_tracker)?;
224 state_machine.prepared_statement = Some(prepared_statement);
225 Ok(state_machine)
226 }
227
228 pub fn prepared_statement(&self) -> Option<&ExtStatement> {
229 self.prepared_statement.as_ref()
230 }
231
232 pub fn query_tracker(&self) -> Option<&QueryExecutionTracker> {
233 self.query_tracker.as_ref()
234 }
235
236 pub fn tracker_matches_session(&self) -> bool {
237 self.query_tracker
238 .as_ref()
239 .is_some_and(|query_tracker| self.session.is_bound_to(query_tracker))
240 }
241
242 pub fn begin_analyze(&self) {
243 self.record_phase_timestamp("analyze", "start");
244 self.translate_to(QueryState::RUNNING(RUNNING::ANALYZING));
245 }
246
247 pub fn end_analyze(&self) {
248 self.record_phase_timestamp("analyze", "end");
249 }
250
251 pub fn begin_optimize(&self) {
252 self.record_phase_timestamp("optimize", "start");
253 self.translate_to(QueryState::RUNNING(RUNNING::OPTIMIZING));
254 }
255
256 pub fn end_optimize(&self) {
257 self.record_phase_timestamp("optimize", "end");
258 }
259
260 pub fn begin_schedule(&self) {
261 self.record_phase_timestamp("schedule", "start");
262 self.translate_to(QueryState::RUNNING(RUNNING::SCHEDULING));
263 }
264
265 pub fn end_schedule(&self) {
266 self.record_phase_timestamp("schedule", "end");
267 }
268
269 pub fn finish(&self) {
270 self.translate_to(QueryState::DONE(DONE::FINISHED));
271 }
272
273 pub fn cancel(&self) {
274 self.translate_to(QueryState::DONE(DONE::CANCELLED));
275 }
276
277 pub fn fail(&self) {
278 self.translate_to(QueryState::DONE(DONE::FAILED));
279 }
280
281 pub fn state(&self) -> QueryState {
282 self.state.read().clone()
283 }
284
285 pub fn duration(&self) -> Duration {
286 self.start.elapsed()
287 }
288
289 fn translate_to(&self, state: QueryState) {
290 *self.state.write() = state;
291 }
292}
293
294#[derive(Debug, Clone)]
295pub enum QueryState {
296 ACCEPTING,
297 RUNNING(RUNNING),
298 DONE(DONE),
299}
300
301impl AsRef<str> for QueryState {
302 fn as_ref(&self) -> &str {
303 match self {
304 QueryState::ACCEPTING => "ACCEPTING",
305 QueryState::RUNNING(e) => e.as_ref(),
306 QueryState::DONE(e) => e.as_ref(),
307 }
308 }
309}
310
311#[derive(Debug, Clone)]
312pub enum RUNNING {
313 DISPATCHING,
314 ANALYZING,
315 OPTIMIZING,
316 SCHEDULING,
317}
318
319impl AsRef<str> for RUNNING {
320 fn as_ref(&self) -> &str {
321 match self {
322 Self::DISPATCHING => "DISPATCHING",
323 Self::ANALYZING => "ANALYZING",
324 Self::OPTIMIZING => "OPTIMIZING",
325 Self::SCHEDULING => "SCHEDULING",
326 }
327 }
328}
329
330#[derive(Debug, Clone)]
331pub enum DONE {
332 FINISHED,
333 FAILED,
334 CANCELLED,
335}
336
337impl AsRef<str> for DONE {
338 fn as_ref(&self) -> &str {
339 match self {
340 Self::FINISHED => "FINISHED",
341 Self::FAILED => "FAILED",
342 Self::CANCELLED => "CANCELLED",
343 }
344 }
345}