Skip to main content

sqlx_postgres/connection/
executor.rs

1use crate::error::Error;
2use crate::executor::{Execute, Executor};
3use crate::io::{PortalId, StatementId};
4use crate::logger::QueryLogger;
5use crate::message::{
6    self, BackendMessageFormat, Bind, Close, CommandComplete, DataRow, ParameterDescription, Parse,
7    ParseComplete, RowDescription,
8};
9use crate::statement::PgStatementMetadata;
10use crate::{
11    statement::PgStatement, PgArguments, PgConnection, PgDatabaseError, PgQueryResult, PgRow,
12    PgTypeInfo, PgValueFormat, Postgres,
13};
14use futures_core::future::BoxFuture;
15use futures_core::stream::BoxStream;
16use futures_core::Stream;
17use futures_util::TryStreamExt;
18use sqlx_core::arguments::Arguments;
19use sqlx_core::sql_str::SqlStr;
20use sqlx_core::Either;
21use std::{pin::pin, sync::Arc};
22
23async fn prepare(
24    conn: &mut PgConnection,
25    sql: &str,
26    arg_types: &[PgTypeInfo],
27    metadata: Option<Arc<PgStatementMetadata>>,
28    persistent: bool,
29    resolve_column_origin: bool,
30) -> Result<(StatementId, Arc<PgStatementMetadata>), Error> {
31    // if cache is disabled, persistent statements get an id but are never evicted
32    // which causes a memory leak
33    let id = if persistent && conn.inner.cache_statement.is_enabled() {
34        let id = conn.inner.next_statement_id;
35        conn.inner.next_statement_id = id.next();
36        id
37    } else {
38        StatementId::UNNAMED
39    };
40
41    // build a list of type OIDs to send to the database in the PARSE command
42    // we have not yet started the query sequence, so we are *safe* to cleanly make
43    // additional queries here to get any missing OIDs
44    let param_types = conn.resolve_types(arg_types).await?;
45
46    // flush and wait until we are re-ready
47    conn.wait_until_ready().await?;
48
49    // next we send the PARSE command to the server
50    conn.inner.stream.write_msg(Parse {
51        param_types: &param_types,
52        query: sql,
53        statement: id,
54    })?;
55
56    if metadata.is_none() {
57        // get the statement columns and parameters
58        conn.inner
59            .stream
60            .write_msg(message::Describe::Statement(id))?;
61    }
62
63    // we ask for the server to immediately send us the result of the PARSE command
64    conn.write_sync();
65    conn.inner.stream.flush().await?;
66
67    // indicates that the SQL query string is now successfully parsed and has semantic validity
68    conn.inner.stream.recv_expect::<ParseComplete>().await?;
69
70    let (metadata, ran_catalog_query) = if let Some(metadata) = metadata {
71        // each SYNC produces one READY FOR QUERY
72        conn.recv_ready_for_query().await?;
73
74        // we already have metadata
75        (metadata, false)
76    } else {
77        let parameters = recv_desc_params(conn).await?;
78
79        let row_desc = recv_desc_rows(conn).await?;
80
81        // each SYNC produces one READY FOR QUERY
82        conn.recv_ready_for_query().await?;
83
84        let (metadata, ran_catalog_query) = conn
85            .resolve_statement_metadata::<true>(Some(parameters), row_desc, resolve_column_origin)
86            .await?;
87
88        // ensure that if we did fetch custom data, we wait until we are fully ready before
89        // continuing
90        conn.wait_until_ready().await?;
91
92        (metadata, ran_catalog_query)
93    };
94
95    // Resolving custom-type OIDs above goes through the simple query protocol, which
96    // destroys the unnamed prepared statement. Re-parse it so the Bind that follows can
97    // reference it; named (persistent) statements are unaffected. See #4305.
98    if !persistent && ran_catalog_query {
99        conn.inner.stream.write_msg(Parse {
100            param_types: &param_types,
101            query: sql,
102            statement: id,
103        })?;
104        conn.write_sync();
105        conn.inner.stream.flush().await?;
106        conn.inner.stream.recv_expect::<ParseComplete>().await?;
107        conn.recv_ready_for_query().await?;
108    }
109
110    Ok((id, metadata))
111}
112
113async fn recv_desc_params(conn: &mut PgConnection) -> Result<ParameterDescription, Error> {
114    conn.inner.stream.recv_expect().await
115}
116
117async fn recv_desc_rows(conn: &mut PgConnection) -> Result<Option<RowDescription>, Error> {
118    let rows: Option<RowDescription> = match conn.inner.stream.recv().await? {
119        // describes the rows that will be returned when the statement is eventually executed
120        message if message.format == BackendMessageFormat::RowDescription => {
121            Some(message.decode()?)
122        }
123
124        // no data would be returned if this statement was executed
125        message if message.format == BackendMessageFormat::NoData => None,
126
127        message => {
128            return Err(err_protocol!(
129                "expecting RowDescription or NoData but received {:?}",
130                message.format
131            ));
132        }
133    };
134
135    Ok(rows)
136}
137
138impl PgConnection {
139    // wait for CloseComplete to indicate a statement was closed
140    pub(super) async fn wait_for_close_complete(&mut self, mut count: usize) -> Result<(), Error> {
141        // we need to wait for the [CloseComplete] to be returned from the server
142        while count > 0 {
143            match self.inner.stream.recv().await? {
144                message if message.format == BackendMessageFormat::PortalSuspended => {
145                    // there was an open portal
146                    // this can happen if the last time a statement was used it was not fully executed
147                }
148
149                message if message.format == BackendMessageFormat::CloseComplete => {
150                    // successfully closed the statement (and freed up the server resources)
151                    count -= 1;
152                }
153
154                message => {
155                    return Err(err_protocol!(
156                        "expecting PortalSuspended or CloseComplete but received {:?}",
157                        message.format
158                    ));
159                }
160            }
161        }
162
163        Ok(())
164    }
165
166    #[inline(always)]
167    pub(crate) fn write_sync(&mut self) {
168        self.inner
169            .stream
170            .write_msg(message::Sync)
171            .expect("BUG: Sync should not be too big for protocol");
172
173        // all SYNC messages will return a ReadyForQuery
174        self.inner.pending_ready_for_query_count += 1;
175    }
176
177    async fn get_or_prepare(
178        &mut self,
179        sql: &str,
180        parameters: &[PgTypeInfo],
181        persistent: bool,
182        // optional metadata that was provided by the user, this means they are reusing
183        // a statement object
184        metadata: Option<Arc<PgStatementMetadata>>,
185        resolve_column_origin: bool,
186    ) -> Result<(StatementId, Arc<PgStatementMetadata>), Error> {
187        if let Some(statement) = self.inner.cache_statement.get_mut(sql) {
188            return Ok((*statement).clone());
189        }
190
191        let statement = prepare(
192            self,
193            sql,
194            parameters,
195            metadata,
196            persistent,
197            resolve_column_origin,
198        )
199        .await?;
200
201        if persistent && self.inner.cache_statement.is_enabled() {
202            if let Some((id, _)) = self.inner.cache_statement.insert(sql, statement.clone()) {
203                self.inner.stream.write_msg(Close::Statement(id))?;
204                self.write_sync();
205
206                self.inner.stream.flush().await?;
207
208                self.wait_for_close_complete(1).await?;
209                self.recv_ready_for_query().await?;
210            }
211        }
212
213        Ok(statement)
214    }
215
216    async fn try_get_or_prepare<'e, 'c: 'e, 'q: 'e>(
217        &'c mut self,
218        sql: &str,
219        arguments: Option<&mut PgArguments>,
220        persistent: bool,
221        metadata_opt: Option<Arc<PgStatementMetadata>>,
222    ) -> Result<(PgValueFormat, Arc<PgStatementMetadata>), Error> {
223        let metadata: Arc<PgStatementMetadata>;
224
225        let format = if let Some(arguments) = arguments {
226            // Check this before we write anything to the stream.
227            //
228            // Note: Postgres actually interprets this value as unsigned,
229            // making the max number of parameters 65535, not 32767
230            // https://github.com/launchbadge/sqlx/issues/3464
231            // https://www.postgresql.org/docs/current/limits.html
232            let num_params = u16::try_from(arguments.len()).map_err(|_| {
233                err_protocol!(
234                    "PgConnection::run(): too many arguments for query: {}",
235                    arguments.len()
236                )
237            })?;
238
239            // prepare the statement if this our first time executing it
240            // always return the statement ID here
241            let (statement, metadata_) = self
242                .get_or_prepare(sql, &arguments.types, persistent, metadata_opt, false)
243                .await?;
244
245            metadata = metadata_;
246
247            // patch holes created during encoding
248            arguments.apply_patches(self, &metadata.parameters).await?;
249
250            // consume messages till `ReadyForQuery` before bind and execute
251            self.wait_until_ready().await?;
252
253            // bind to attach the arguments to the statement and create a portal
254            self.inner.stream.write_msg(Bind {
255                portal: PortalId::UNNAMED,
256                statement,
257                formats: &[PgValueFormat::Binary],
258                num_params,
259                params: &arguments.buffer,
260                result_formats: &[PgValueFormat::Binary],
261            })?;
262
263            // executes the portal up to the passed limit
264            // the protocol-level limit acts nearly identically to the `LIMIT` in SQL
265            self.inner.stream.write_msg(message::Execute {
266                portal: PortalId::UNNAMED,
267                // Non-zero limits cause query plan pessimization by disabling parallel workers:
268                // https://github.com/launchbadge/sqlx/issues/3673
269                limit: 0,
270            })?;
271            // From https://www.postgresql.org/docs/current/protocol-flow.html:
272            //
273            // "An unnamed portal is destroyed at the end of the transaction, or as
274            // soon as the next Bind statement specifying the unnamed portal as
275            // destination is issued. (Note that a simple Query message also
276            // destroys the unnamed portal."
277
278            // we ask the database server to close the unnamed portal and free the associated resources
279            // earlier - after the execution of the current query.
280            self.inner
281                .stream
282                .write_msg(Close::Portal(PortalId::UNNAMED))?;
283
284            // finally, [Sync] asks postgres to process the messages that we sent and respond with
285            // a [ReadyForQuery] message when it's completely done. Theoretically, we could send
286            // dozens of queries before a [Sync] and postgres can handle that. Execution on the server
287            // is still serial but it would reduce round-trips. Some kind of builder pattern that is
288            // termed batching might suit this.
289            self.write_sync();
290
291            // prepared statements are binary
292            PgValueFormat::Binary
293        } else {
294            // Query will trigger a ReadyForQuery
295            self.queue_simple_query(sql)?;
296
297            // metadata starts out as "nothing"
298            metadata = Arc::new(PgStatementMetadata::default());
299
300            // and unprepared statements are text
301            PgValueFormat::Text
302        };
303
304        self.inner.stream.flush().await?;
305
306        Ok((format, metadata))
307    }
308
309    pub(crate) async fn run<'e, 'c: 'e, 'q: 'e>(
310        &'c mut self,
311        query: SqlStr,
312        mut arguments: Option<PgArguments>,
313        persistent: bool,
314        metadata_opt: Option<Arc<PgStatementMetadata>>,
315    ) -> Result<impl Stream<Item = Result<Either<PgQueryResult, PgRow>, Error>> + 'e, Error> {
316        let mut logger = QueryLogger::new(query, self.inner.log_settings.clone());
317        let sql = logger.sql().as_str();
318
319        // before we continue, wait until we are "ready" to accept more queries
320        self.wait_until_ready().await?;
321
322        let (mut format, mut metadata) = self
323            .try_get_or_prepare(sql, arguments.as_mut(), persistent, metadata_opt.clone())
324            .await?;
325
326        let mut message = match self.inner.stream.recv().await {
327            Ok(msg) => msg,
328            Err(err) => {
329                if let Some(clear_backend_cache) = check_stale_plan(&err) {
330                    // Save transaction mode. It will be lost after invalidating
331                    let is_in_tx = self.in_transaction();
332
333                    self.invalidate_cached_statement(sql, clear_backend_cache)
334                        .await?;
335
336                    // If we were in transaction mode we can't retry statement,
337                    //    so we can immediately return err
338                    if is_in_tx {
339                        return Err(err);
340                    }
341
342                    // Otherwise we can retry statement in hope everything is ok.
343                    (format, metadata) = self
344                        .try_get_or_prepare(
345                            sql,
346                            // It should be safe to retry `patch` on the same arguments
347                            arguments.as_mut(),
348                            persistent,
349                            metadata_opt.clone(),
350                        )
351                        .await?;
352
353                    self.inner.stream.recv().await?
354                } else {
355                    return Err(err);
356                }
357            }
358        };
359
360        Ok(try_stream! {
361            loop {
362                match message.format {
363                    BackendMessageFormat::BindComplete
364                    | BackendMessageFormat::ParseComplete
365                    | BackendMessageFormat::ParameterDescription
366                    | BackendMessageFormat::NoData
367                    // unnamed portal has been closed
368                    | BackendMessageFormat::CloseComplete
369                    => {
370                        // harmless messages to ignore
371                    }
372
373                    // "Execute phase is always terminated by the appearance of
374                    // exactly one of these messages: CommandComplete,
375                    // EmptyQueryResponse (if the portal was created from an
376                    // empty query string), ErrorResponse, or PortalSuspended"
377                    BackendMessageFormat::CommandComplete => {
378                        // a SQL command completed normally
379                        let cc: CommandComplete = message.decode()?;
380
381                        let rows_affected = cc.rows_affected();
382                        logger.increase_rows_affected(rows_affected);
383                        r#yield!(Either::Left(PgQueryResult {
384                            rows_affected,
385                        }));
386                    }
387
388                    BackendMessageFormat::EmptyQueryResponse => {
389                        // empty query string passed to an unprepared execute
390                    }
391
392                    // Message::ErrorResponse is handled in self.stream.recv()
393
394                    // incomplete query execution has finished
395                    BackendMessageFormat::PortalSuspended => {}
396
397                    // indicates that a *new* set of rows are about to be returned
398                    BackendMessageFormat::RowDescription => {
399                        let (new_metadata, _) = self.resolve_statement_metadata::<false>(
400                            None,
401                            Some(message.decode()?),
402                            false,
403                        ).await?;
404
405                        metadata = new_metadata;
406                    }
407
408                    BackendMessageFormat::DataRow => {
409                        logger.increment_rows_returned();
410
411                        // one of the set of rows returned by a SELECT, FETCH, etc query
412                        let data: DataRow = message.decode()?;
413                        let row = PgRow {
414                            data,
415                            format,
416                            metadata: Arc::clone(&metadata),
417                        };
418
419                        r#yield!(Either::Right(row));
420                    }
421
422                    BackendMessageFormat::ReadyForQuery => {
423                        // processing of the query string is complete
424                        self.handle_ready_for_query(message)?;
425                        break;
426                    }
427
428                    _ => {
429                        return Err(err_protocol!(
430                            "execute: unexpected message: {:?}",
431                            message.format
432                        ));
433                    }
434                }
435
436                message = self.inner.stream.recv().await?;
437            }
438
439            Ok(())
440        })
441    }
442}
443
444impl<'c> Executor<'c> for &'c mut PgConnection {
445    type Database = Postgres;
446
447    fn fetch_many<'e, 'q, E>(
448        self,
449        mut query: E,
450    ) -> BoxStream<'e, Result<Either<PgQueryResult, PgRow>, Error>>
451    where
452        'c: 'e,
453        E: Execute<'q, Self::Database>,
454        'q: 'e,
455        E: 'q,
456    {
457        // False positive: https://github.com/rust-lang/rust-clippy/issues/12560
458        #[allow(clippy::map_clone)]
459        let metadata = query.statement().map(|s| Arc::clone(&s.metadata));
460        let arguments = query.take_arguments().map_err(Error::Encode);
461        let persistent = query.persistent();
462        let sql = query.sql();
463
464        Box::pin(try_stream! {
465            let arguments = arguments?;
466            let mut s = pin!(self.run(sql, arguments, persistent, metadata).await?);
467
468            while let Some(v) = s.try_next().await? {
469                r#yield!(v);
470            }
471
472            Ok(())
473        })
474    }
475
476    fn fetch_optional<'e, 'q, E>(self, mut query: E) -> BoxFuture<'e, Result<Option<PgRow>, Error>>
477    where
478        'c: 'e,
479        E: Execute<'q, Self::Database>,
480        'q: 'e,
481        E: 'q,
482    {
483        // False positive: https://github.com/rust-lang/rust-clippy/issues/12560
484        #[allow(clippy::map_clone)]
485        let metadata = query.statement().map(|s| Arc::clone(&s.metadata));
486        let arguments = query.take_arguments().map_err(Error::Encode);
487        let persistent = query.persistent();
488
489        Box::pin(async move {
490            let sql = query.sql();
491            let arguments = arguments?;
492            let mut s = pin!(self.run(sql, arguments, persistent, metadata).await?);
493
494            // With deferred constraints we need to check all responses as we
495            // could get a OK response (with uncommitted data), only to get an
496            // error response after (when the deferred constraint is actually
497            // checked).
498            let mut ret = None;
499            while let Some(result) = s.try_next().await? {
500                match result {
501                    Either::Right(r) if ret.is_none() => ret = Some(r),
502                    _ => {}
503                }
504            }
505            Ok(ret)
506        })
507    }
508
509    fn prepare_with<'e>(
510        self,
511        sql: SqlStr,
512        parameters: &'e [PgTypeInfo],
513    ) -> BoxFuture<'e, Result<PgStatement, Error>>
514    where
515        'c: 'e,
516    {
517        Box::pin(async move {
518            self.wait_until_ready().await?;
519
520            let (_, metadata) = self
521                .get_or_prepare(sql.as_str(), parameters, true, None, true)
522                .await?;
523
524            Ok(PgStatement { sql, metadata })
525        })
526    }
527
528    #[cfg(feature = "offline")]
529    fn describe<'e>(
530        self,
531        sql: SqlStr,
532    ) -> BoxFuture<'e, Result<crate::describe::Describe<Self::Database>, Error>>
533    where
534        'c: 'e,
535    {
536        Box::pin(async move {
537            self.wait_until_ready().await?;
538
539            let (stmt_id, metadata) = self
540                .get_or_prepare(sql.as_str(), &[], true, None, true)
541                .await?;
542
543            let nullable = self.get_nullable_for_columns(stmt_id, &metadata).await?;
544
545            Ok(crate::describe::Describe {
546                columns: metadata.columns.clone(),
547                nullable,
548                parameters: Some(Either::Left(metadata.parameters.clone())),
549            })
550        })
551    }
552}
553
554// Returns:
555// - `None`        - if not 'stale query plan'
556// - `Some(false)` - if it is stale plan, but we don't need to deallocate objects on backend.
557//                   It can happen because of `DISCARD ALL`, `DEALLOCATE` or due to pgbouncer in
558//                      transaction pooling mode
559// - `Some(true)`  - if we should invalidate both backend and frontend caches
560fn check_stale_plan(error: &Error) -> Option<bool> {
561    let error = error
562        .as_database_error()?
563        .try_downcast_ref::<PgDatabaseError>()?;
564
565    match (error.code(), error.routine()) {
566        // "cached plan must not change result type"
567        ("0A000", Some("RevalidateCachedQuery")) => Some(true),
568        // DISCARD ALL / DEALLOCATE / pgbouncer
569        ("26000", _) => Some(false),
570        _ => None,
571    }
572}