Skip to main content

diesel_async/pg/
mod.rs

1//! Provides types and functions related to working with PostgreSQL
2//!
3//! Much of this module is re-exported from database agnostic locations.
4//! However, if you are writing code specifically to extend Diesel on
5//! PostgreSQL, you may need to work with this module directly.
6
7use self::error_helper::ErrorHelper;
8use self::row::PgRow;
9use self::serialize::ToSqlHelper;
10use crate::AsyncMultiConnectionHelper;
11use crate::stmt_cache::{CallbackHelper, QueryFragmentHelper, StatementHelper};
12use crate::{AnsiTransactionManager, AsyncConnection, AsyncConnectionCore, SimpleAsyncConnection};
13use diesel::connection::StrQueryHelper;
14use diesel::connection::statement_cache::{
15    PrepareForCache, PreparedStatement, QueryFragmentForCachedStatement, StatementCache,
16};
17use diesel::connection::{CacheSize, Instrumentation};
18use diesel::connection::{DynInstrumentation, InstrumentationEvent};
19use diesel::pg::{
20    Pg, PgMetadataCache, PgMetadataCacheKey, PgMetadataLookup, PgQueryBuilder, PgTypeMetadata,
21};
22use diesel::query_builder::bind_collector::RawBytesBindCollector;
23use diesel::query_builder::{AsQuery, QueryBuilder, QueryFragment, QueryId};
24use diesel::result::{DatabaseErrorKind, Error};
25use diesel::sql_types::TypeMetadata;
26use diesel::{ConnectionError, ConnectionResult, QueryResult};
27use futures_core::future::BoxFuture;
28use futures_core::stream::BoxStream;
29use futures_util::TryFutureExt;
30use futures_util::future::Either;
31use futures_util::stream::TryStreamExt;
32use futures_util::{FutureExt, StreamExt};
33use std::collections::{HashMap, HashSet};
34use std::future::Future;
35use std::sync::Arc;
36use tokio::sync::{Mutex, broadcast, mpsc, oneshot};
37use tokio_postgres::Statement;
38use tokio_postgres::types::ToSql;
39use tokio_postgres::types::Type;
40
41pub use self::transaction_builder::TransactionBuilder;
42
43mod error_helper;
44mod row;
45mod serialize;
46mod transaction_builder;
47
48const FAKE_OID: u32 = 0;
49
50/// A connection to a PostgreSQL database.
51///
52/// Connection URLs should be in the form
53/// `postgres://[user[:password]@]host/database_name`
54///
55/// Checkout the documentation of the [tokio_postgres]
56/// crate for details about the format
57///
58/// [tokio_postgres]: https://docs.rs/tokio-postgres/0.7.6/tokio_postgres/config/struct.Config.html#url
59///
60/// ## Pipelining
61///
62/// This connection supports *pipelined* requests. Pipelining can improve performance in use cases in which multiple,
63/// independent queries need to be executed. In a traditional workflow, each query is sent to the server after the
64/// previous query completes. In contrast, pipelining allows the client to send all of the queries to the server up
65/// front, minimizing time spent by one side waiting for the other to finish sending data:
66///
67/// ```not_rust
68///             Sequential                              Pipelined
69/// | Client         | Server          |    | Client         | Server          |
70/// |----------------|-----------------|    |----------------|-----------------|
71/// | send query 1   |                 |    | send query 1   |                 |
72/// |                | process query 1 |    | send query 2   | process query 1 |
73/// | receive rows 1 |                 |    | send query 3   | process query 2 |
74/// | send query 2   |                 |    | receive rows 1 | process query 3 |
75/// |                | process query 2 |    | receive rows 2 |                 |
76/// | receive rows 2 |                 |    | receive rows 3 |                 |
77/// | send query 3   |                 |
78/// |                | process query 3 |
79/// | receive rows 3 |                 |
80/// ```
81///
82/// In both cases, the PostgreSQL server is executing the queries **sequentially** - pipelining just allows both sides of
83/// the connection to work concurrently when possible.
84///
85/// Pipelining happens automatically when futures are polled concurrently (for example, by using the futures `join`
86/// combinator):
87///
88/// ```rust
89/// # include!("../doctest_setup.rs");
90/// use diesel_async::RunQueryDsl;
91///
92/// #
93/// # #[tokio::main(flavor = "current_thread")]
94/// # async fn main() {
95/// #     run_test().await.unwrap();
96/// # }
97/// #
98/// # async fn run_test() -> QueryResult<()> {
99/// #     use diesel::sql_types::{Text, Integer};
100/// #     let conn = establish_connection().await;
101///       let q1 = diesel::select(1_i32.into_sql::<Integer>());
102///       let q2 = diesel::select(2_i32.into_sql::<Integer>());
103///
104///       // construct multiple futures for different queries
105///       let f1 = q1.get_result::<i32>(&mut &conn);
106///       let f2 = q2.get_result::<i32>(&mut &conn);
107///
108///       // wait on both results
109///       let res = futures_util::try_join!(f1, f2)?;
110///
111///       assert_eq!(res.0, 1);
112///       assert_eq!(res.1, 2);
113///       # Ok(())
114/// # }
115/// ```
116///
117/// For more complex cases, an immutable reference to the connection need to be used:
118/// ```rust
119/// # include!("../doctest_setup.rs");
120/// use diesel_async::RunQueryDsl;
121///
122/// #
123/// # #[tokio::main(flavor = "current_thread")]
124/// # async fn main() {
125/// #     run_test().await.unwrap();
126/// # }
127/// #
128/// # async fn run_test() -> QueryResult<()> {
129/// #     use diesel::sql_types::{Text, Integer};
130/// #     let conn = &mut establish_connection().await;
131/// #
132///       async fn fn12(mut conn: &AsyncPgConnection) -> QueryResult<(i32, i32)> {
133///           let f1 = diesel::select(1_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
134///           let f2 = diesel::select(2_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
135///
136///           futures_util::try_join!(f1, f2)
137///       }
138///
139///       async fn fn34(mut conn: &AsyncPgConnection) -> QueryResult<(i32, i32)> {
140///           let f3 = diesel::select(3_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
141///           let f4 = diesel::select(4_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
142///
143///           futures_util::try_join!(f3, f4)
144///       }
145///
146///       let f12 = fn12(&conn);
147///       let f34 = fn34(&conn);
148///
149///       let ((r1, r2), (r3, r4)) = futures_util::try_join!(f12, f34).unwrap();
150///
151///       assert_eq!(r1, 1);
152///       assert_eq!(r2, 2);
153///       assert_eq!(r3, 3);
154///       assert_eq!(r4, 4);
155///       # Ok(())
156/// # }
157/// ```
158///
159/// ## TLS
160///
161/// Connections created by [`AsyncPgConnection::establish`] do not support TLS.
162///
163/// TLS support for tokio_postgres connections is implemented by external crates, e.g. [tokio_postgres_rustls].
164///
165/// [`AsyncPgConnection::try_from_client_and_connection`] can be used to construct a connection from an existing
166/// [`tokio_postgres::Connection`] with TLS enabled.
167///
168/// [tokio_postgres_rustls]: https://docs.rs/tokio-postgres-rustls/0.12.0/tokio_postgres_rustls/
169pub struct AsyncPgConnection {
170    conn: tokio_postgres::Client,
171    stmt_cache: Mutex<StatementCache<diesel::pg::Pg, StatementHelper<Statement>>>,
172    transaction_state: Mutex<AnsiTransactionManager>,
173    metadata_cache: Mutex<PgMetadataCache>,
174    connection_future: Option<broadcast::Receiver<Arc<tokio_postgres::Error>>>,
175    notification_rx: Option<mpsc::UnboundedReceiver<QueryResult<diesel::pg::PgNotification>>>,
176    shutdown_channel: Option<oneshot::Sender<()>>,
177    // a sync mutex is fine here as we only hold it for a really short time
178    instrumentation: Arc<std::sync::Mutex<DynInstrumentation>>,
179}
180
181impl SimpleAsyncConnection for AsyncPgConnection {
182    async fn batch_execute(&mut self, query: &str) -> QueryResult<()> {
183        SimpleAsyncConnection::batch_execute(&mut &*self, query).await
184    }
185}
186
187impl SimpleAsyncConnection for &AsyncPgConnection {
188    async fn batch_execute(&mut self, query: &str) -> QueryResult<()> {
189        self.record_instrumentation(InstrumentationEvent::start_query(&StrQueryHelper::new(
190            query,
191        )));
192        let connection_future = self.connection_future.as_ref().map(|rx| rx.resubscribe());
193        let batch_execute = self
194            .conn
195            .batch_execute(query)
196            .map_err(ErrorHelper)
197            .map_err(Into::into);
198
199        let r = drive_future(connection_future, batch_execute).await;
200        let r = {
201            let mut transaction_manager = self.transaction_state.lock().await;
202            update_transaction_manager_status(r, &mut transaction_manager)
203        };
204        self.record_instrumentation(InstrumentationEvent::finish_query(
205            &StrQueryHelper::new(query),
206            r.as_ref().err(),
207        ));
208        r
209    }
210}
211
212impl AsyncConnectionCore for AsyncPgConnection {
213    // The returned future must not outlive the connection it borrows.
214    type LoadFuture<'conn, 'query> = BoxFuture<'conn, QueryResult<Self::Stream<'conn, 'query>>>;
215    type ExecuteFuture<'conn, 'query> = BoxFuture<'conn, QueryResult<usize>>;
216    type Stream<'conn, 'query> = BoxStream<'static, QueryResult<PgRow>>;
217    type Row<'conn, 'query> = PgRow;
218    type Backend = diesel::pg::Pg;
219
220    fn load<'conn, 'query, T>(&'conn mut self, source: T) -> Self::LoadFuture<'conn, 'query>
221    where
222        T: AsQuery + 'query,
223        T::Query: QueryFragment<Self::Backend> + QueryId + 'query,
224    {
225        let query = source.as_query();
226        let load_future = self.with_prepared_statement(query, load_prepared);
227
228        self.run_with_connection_future(load_future)
229    }
230
231    fn execute_returning_count<'conn, 'query, T>(
232        &'conn mut self,
233        source: T,
234    ) -> Self::ExecuteFuture<'conn, 'query>
235    where
236        T: QueryFragment<Self::Backend> + QueryId + 'query,
237    {
238        let execute = self.with_prepared_statement(source, execute_prepared);
239        self.run_with_connection_future(execute)
240    }
241}
242
243// Enable query pipelining via shared references while binding futures
244// to the lifetime of the borrowed connection.
245impl<'a> AsyncConnectionCore for &'a AsyncPgConnection {
246    type LoadFuture<'conn, 'query> = BoxFuture<'a, QueryResult<Self::Stream<'conn, 'query>>>;
247    type ExecuteFuture<'conn, 'query> = BoxFuture<'a, QueryResult<usize>>;
248    type Stream<'conn, 'query> = BoxStream<'static, QueryResult<PgRow>>;
249    type Row<'conn, 'query> = PgRow;
250    type Backend = diesel::pg::Pg;
251
252    fn load<'conn, 'query, T>(&'conn mut self, source: T) -> Self::LoadFuture<'conn, 'query>
253    where
254        T: AsQuery + 'query,
255        T::Query: QueryFragment<Self::Backend> + QueryId + 'query,
256    {
257        let query = source.as_query();
258        let load_future = self.with_prepared_statement(query, load_prepared);
259
260        self.run_with_connection_future(load_future)
261    }
262
263    fn execute_returning_count<'conn, 'query, T>(
264        &'conn mut self,
265        source: T,
266    ) -> Self::ExecuteFuture<'conn, 'query>
267    where
268        T: QueryFragment<Self::Backend> + QueryId + 'query,
269    {
270        let execute = self.with_prepared_statement(source, execute_prepared);
271        self.run_with_connection_future(execute)
272    }
273}
274
275impl AsyncConnection for AsyncPgConnection {
276    type TransactionManager = AnsiTransactionManager;
277
278    async fn establish(database_url: &str) -> ConnectionResult<Self> {
279        let mut instrumentation = DynInstrumentation::default_instrumentation();
280        instrumentation.on_connection_event(InstrumentationEvent::start_establish_connection(
281            database_url,
282        ));
283        let instrumentation = Arc::new(std::sync::Mutex::new(instrumentation));
284        let (client, connection) = tokio_postgres::connect(database_url, tokio_postgres::NoTls)
285            .await
286            .map_err(ErrorHelper)?;
287
288        let (error_rx, notification_rx, shutdown_tx) = drive_connection(connection);
289
290        let r = Self::setup(
291            client,
292            Some(error_rx),
293            Some(notification_rx),
294            Some(shutdown_tx),
295            Arc::clone(&instrumentation),
296        )
297        .await;
298
299        instrumentation
300            .lock()
301            .unwrap_or_else(|e| e.into_inner())
302            .on_connection_event(InstrumentationEvent::finish_establish_connection(
303                database_url,
304                r.as_ref().err(),
305            ));
306        r
307    }
308
309    fn transaction_state(&mut self) -> &mut AnsiTransactionManager {
310        self.transaction_state.get_mut()
311    }
312
313    fn instrumentation(&mut self) -> &mut dyn Instrumentation {
314        // there should be no other pending future when this is called
315        // that means there is only one instance of this arc and
316        // we can simply access the inner data
317        if let Some(instrumentation) = Arc::get_mut(&mut self.instrumentation) {
318            &mut **(instrumentation.get_mut().unwrap_or_else(|p| p.into_inner()))
319        } else {
320            panic!("Cannot access shared instrumentation")
321        }
322    }
323
324    fn set_instrumentation(&mut self, instrumentation: impl Instrumentation) {
325        self.instrumentation = Arc::new(std::sync::Mutex::new(instrumentation.into()));
326    }
327
328    fn set_prepared_statement_cache_size(&mut self, size: CacheSize) {
329        self.stmt_cache.get_mut().set_cache_size(size)
330    }
331}
332
333impl Drop for AsyncPgConnection {
334    fn drop(&mut self) {
335        if let Some(tx) = self.shutdown_channel.take() {
336            let _ = tx.send(());
337        }
338    }
339}
340
341async fn load_prepared(
342    conn: &tokio_postgres::Client,
343    stmt: Statement,
344    binds: Vec<ToSqlHelper>,
345) -> QueryResult<BoxStream<'static, QueryResult<PgRow>>> {
346    let res = conn.query_raw(&stmt, binds).await.map_err(ErrorHelper)?;
347
348    Ok(res
349        .map_err(|e| diesel::result::Error::from(ErrorHelper(e)))
350        .map_ok(PgRow::new)
351        .boxed())
352}
353
354async fn execute_prepared(
355    conn: &tokio_postgres::Client,
356    stmt: Statement,
357    binds: Vec<ToSqlHelper>,
358) -> QueryResult<usize> {
359    let binds = binds
360        .iter()
361        .map(|b| b as &(dyn ToSql + Sync))
362        .collect::<Vec<_>>();
363
364    let res = tokio_postgres::Client::execute(conn, &stmt, &binds as &[_])
365        .await
366        .map_err(ErrorHelper)?;
367    res.try_into()
368        .map_err(|e| diesel::result::Error::DeserializationError(Box::new(e)))
369}
370
371#[inline(always)]
372fn update_transaction_manager_status<T>(
373    query_result: QueryResult<T>,
374    transaction_manager: &mut AnsiTransactionManager,
375) -> QueryResult<T> {
376    if let Err(diesel::result::Error::DatabaseError(DatabaseErrorKind::SerializationFailure, _)) =
377        query_result
378        && !transaction_manager.is_commit
379    {
380        transaction_manager
381            .status
382            .set_requires_rollback_maybe_up_to_top_level(true);
383    }
384    query_result
385}
386
387fn prepare_statement_helper<'conn>(
388    conn: &'conn tokio_postgres::Client,
389    sql: &str,
390    _is_for_cache: PrepareForCache,
391    metadata: &[PgTypeMetadata],
392) -> CallbackHelper<
393    impl Future<Output = QueryResult<(StatementHelper<Statement>, &'conn tokio_postgres::Client)>>
394    + Send
395    + use<'conn>,
396> {
397    let bind_types = metadata
398        .iter()
399        .map(type_from_oid)
400        .collect::<QueryResult<Vec<_>>>();
401    // ideally we wouldn't clone the SQL string here
402    // but as we usually cache statements anyway
403    // this is a fixed one time const
404    //
405    // The problem with not cloning it is that we then cannot express
406    // the right result lifetime anymore (at least not easily)
407    let sql = sql.to_string();
408    CallbackHelper(async move {
409        let bind_types = bind_types?;
410        let stmt = conn
411            .prepare_typed(&sql, &bind_types)
412            .await
413            .map_err(ErrorHelper);
414        Ok((StatementHelper(stmt?), conn))
415    })
416}
417
418fn type_from_oid(t: &PgTypeMetadata) -> QueryResult<Type> {
419    let oid = t
420        .oid()
421        .map_err(|e| diesel::result::Error::SerializationError(Box::new(e) as _))?;
422
423    if let Some(tpe) = Type::from_oid(oid) {
424        return Ok(tpe);
425    }
426
427    Ok(Type::new(
428        format!("diesel_custom_type_{oid}"),
429        oid,
430        tokio_postgres::types::Kind::Simple,
431        "public".into(),
432    ))
433}
434
435impl AsyncPgConnection {
436    /// Build a transaction, specifying additional details such as isolation level
437    ///
438    /// See [`TransactionBuilder`] for more examples.
439    ///
440    /// [`TransactionBuilder`]: crate::pg::TransactionBuilder
441    ///
442    /// ```rust
443    /// # include!("../doctest_setup.rs");
444    /// #
445    /// # #[tokio::main(flavor = "current_thread")]
446    /// # async fn main() {
447    /// #     run_test().await.unwrap();
448    /// # }
449    /// #
450    /// # async fn run_test() -> QueryResult<()> {
451    /// #     use schema::users::dsl::*;
452    /// #     let conn = &mut connection_no_transaction().await;
453    /// conn.build_transaction()
454    ///     .read_only()
455    ///     .serializable()
456    ///     .deferrable()
457    ///     .run(async |conn| Ok(()))
458    ///     .await
459    /// # }
460    /// ```
461    pub fn build_transaction(&mut self) -> TransactionBuilder<'_, Self> {
462        TransactionBuilder::new(self)
463    }
464
465    /// Construct a new `AsyncPgConnection` instance from an existing [`tokio_postgres::Client`]
466    pub async fn try_from(conn: tokio_postgres::Client) -> ConnectionResult<Self> {
467        Self::setup(
468            conn,
469            None,
470            None,
471            None,
472            Arc::new(std::sync::Mutex::new(
473                DynInstrumentation::default_instrumentation(),
474            )),
475        )
476        .await
477    }
478
479    /// Constructs a new `AsyncPgConnection` from an existing [`tokio_postgres::Client`] and
480    /// [`tokio_postgres::Connection`]
481    pub async fn try_from_client_and_connection<S>(
482        client: tokio_postgres::Client,
483        conn: tokio_postgres::Connection<tokio_postgres::Socket, S>,
484    ) -> ConnectionResult<Self>
485    where
486        S: tokio_postgres::tls::TlsStream + Unpin + Send + 'static,
487    {
488        let (error_rx, notification_rx, shutdown_tx) = drive_connection(conn);
489
490        Self::setup(
491            client,
492            Some(error_rx),
493            Some(notification_rx),
494            Some(shutdown_tx),
495            Arc::new(std::sync::Mutex::new(
496                DynInstrumentation::default_instrumentation(),
497            )),
498        )
499        .await
500    }
501
502    async fn setup(
503        conn: tokio_postgres::Client,
504        connection_future: Option<broadcast::Receiver<Arc<tokio_postgres::Error>>>,
505        notification_rx: Option<mpsc::UnboundedReceiver<QueryResult<diesel::pg::PgNotification>>>,
506        shutdown_channel: Option<oneshot::Sender<()>>,
507        instrumentation: Arc<std::sync::Mutex<DynInstrumentation>>,
508    ) -> ConnectionResult<Self> {
509        let mut conn = Self {
510            conn,
511            stmt_cache: Mutex::new(StatementCache::new()),
512            transaction_state: Mutex::new(AnsiTransactionManager::default()),
513            metadata_cache: Mutex::new(PgMetadataCache::new()),
514            connection_future,
515            notification_rx,
516            shutdown_channel,
517            instrumentation,
518        };
519        conn.set_config_options()
520            .await
521            .map_err(ConnectionError::CouldntSetupConfiguration)?;
522        Ok(conn)
523    }
524
525    /// Constructs a cancellation token that can later be used to request cancellation of a query running on the connection associated with this client.
526    pub fn cancel_token(&self) -> tokio_postgres::CancelToken {
527        self.conn.cancel_token()
528    }
529
530    async fn set_config_options(&mut self) -> QueryResult<()> {
531        use crate::run_query_dsl::RunQueryDsl;
532
533        futures_util::future::try_join(
534            diesel::sql_query("SET TIME ZONE 'UTC'").execute(&mut &*self),
535            diesel::sql_query("SET CLIENT_ENCODING TO 'UTF8'").execute(&mut &*self),
536        )
537        .await?;
538        Ok(())
539    }
540
541    fn run_with_connection_future<'a, R: 'a>(
542        &self,
543        future: impl Future<Output = QueryResult<R>> + Send + 'a,
544    ) -> BoxFuture<'a, QueryResult<R>> {
545        let connection_future = self.connection_future.as_ref().map(|rx| rx.resubscribe());
546        drive_future(connection_future, future).boxed()
547    }
548
549    fn with_prepared_statement<'a, T, F, R>(
550        &'a self,
551        query: T,
552        callback: fn(&'a tokio_postgres::Client, Statement, Vec<ToSqlHelper>) -> F,
553    ) -> BoxFuture<'a, QueryResult<R>>
554    where
555        T: QueryFragment<diesel::pg::Pg> + QueryId,
556        F: Future<Output = QueryResult<R>> + Send + 'a,
557        R: Send,
558    {
559        self.record_instrumentation(InstrumentationEvent::start_query(&diesel::debug_query(
560            &query,
561        )));
562        // we explicitly destruct the query here before going into the async block
563        //
564        // That's required to remove the send bound from `T` as we have translated
565        // the query type to just a string (for the SQL) and a bunch of bytes (for the binds)
566        // which both are `Send`.
567        // We also collect the query id (essentially an integer) and the safe_to_cache flag here
568        // so there is no need to even access the query in the async block below
569        let mut query_builder = PgQueryBuilder::default();
570
571        let bind_data = construct_bind_data(&query);
572
573        // The code that doesn't need the `T` generic parameter is in a separate function to reduce LLVM IR lines
574        self.with_prepared_statement_after_sql_built(
575            callback,
576            query.is_safe_to_cache_prepared(&Pg),
577            T::query_id(),
578            query.to_sql(&mut query_builder, &Pg),
579            query_builder,
580            bind_data,
581        )
582    }
583
584    fn with_prepared_statement_after_sql_built<'a, F, R>(
585        &'a self,
586        callback: fn(&'a tokio_postgres::Client, Statement, Vec<ToSqlHelper>) -> F,
587        is_safe_to_cache_prepared: QueryResult<bool>,
588        query_id: Option<std::any::TypeId>,
589        to_sql_result: QueryResult<()>,
590        query_builder: PgQueryBuilder,
591        bind_data: BindData,
592    ) -> BoxFuture<'a, QueryResult<R>>
593    where
594        F: Future<Output = QueryResult<R>> + Send + 'a,
595        R: Send,
596    {
597        let raw_connection = &self.conn;
598        let stmt_cache = &self.stmt_cache;
599        let metadata_cache = &self.metadata_cache;
600        let tm = &self.transaction_state;
601        let instrumentation = self.instrumentation.clone();
602        let BindData {
603            collect_bind_result,
604            fake_oid_locations,
605            generated_oids,
606            mut bind_collector,
607        } = bind_data;
608
609        async move {
610            let sql = to_sql_result.map(|_| query_builder.finish())?;
611            let res = async {
612            let is_safe_to_cache_prepared = is_safe_to_cache_prepared?;
613            collect_bind_result?;
614            // Check whether we need to resolve some types at all
615            //
616            // If the user doesn't use custom types there is no need
617            // to bother with that at all
618            if let Some(ref unresolved_types) = generated_oids {
619                let metadata_cache = &mut *metadata_cache.lock().await;
620                let mut real_oids = HashMap::new();
621
622                for ((schema, lookup_type_name), (fake_oid, fake_array_oid)) in
623                    unresolved_types
624                {
625                    // for each unresolved item
626                    // we check whether it's already in the cache
627                    // or perform a lookup and insert it into the cache
628                    let cache_key = PgMetadataCacheKey::new(
629                        schema.as_deref().map(Into::into),
630                        lookup_type_name.into(),
631                    );
632                    let real_metadata = if let Some(type_metadata) =
633                        metadata_cache.lookup_type(&cache_key)
634                    {
635                        type_metadata
636                    } else {
637                        let type_metadata =
638                            lookup_type(schema.clone(), lookup_type_name.clone(), raw_connection)
639                                .await?;
640                        metadata_cache.store_type(cache_key, type_metadata);
641
642                        PgTypeMetadata::from_result(Ok(type_metadata))
643                    };
644                    // let (fake_oid, fake_array_oid) = metadata_lookup.fake_oids(index);
645                    let (real_oid, real_array_oid) = unwrap_oids(&real_metadata);
646                    real_oids.extend([(*fake_oid, real_oid), (*fake_array_oid, real_array_oid)]);
647                }
648
649                // Replace fake OIDs with real OIDs in `bind_collector.metadata`
650                for m in &mut bind_collector.metadata {
651                    let (oid, array_oid) = unwrap_oids(m);
652                    *m = PgTypeMetadata::new(
653                        real_oids.get(&oid).copied().unwrap_or(oid),
654                        real_oids.get(&array_oid).copied().unwrap_or(array_oid)
655                    );
656                }
657                // Replace fake OIDs with real OIDs in `bind_collector.binds`
658                for (bind_index, byte_index) in fake_oid_locations {
659                    replace_fake_oid(&mut bind_collector.binds, &real_oids, bind_index, byte_index)
660                        .ok_or_else(|| {
661                            Error::SerializationError(
662                                format!("diesel_async failed to replace a type OID serialized in bind value {bind_index}").into(),
663                            )
664                        })?;
665                }
666            }
667            let stmt = {
668                let mut stmt_cache = stmt_cache.lock().await;
669                let helper = QueryFragmentHelper {
670                    sql: sql.clone(),
671                    safe_to_cache: is_safe_to_cache_prepared,
672                    bind_count: bind_collector.binds.len(),
673                };
674                let instrumentation = Arc::clone(&instrumentation);
675                stmt_cache
676                    .cached_statement_non_generic(
677                        query_id,
678                        &helper,
679                        &Pg,
680                        &bind_collector.metadata,
681                        raw_connection,
682                        prepare_statement_helper,
683                        &mut move |event: InstrumentationEvent<'_>| {
684                            // we wrap this lock into another callback to prevent locking
685                            // the instrumentation longer than necessary
686                            instrumentation.lock().unwrap_or_else(|e| e.into_inner())
687                                .on_connection_event(event);
688                        },
689                    )
690                    .await?
691                    .0
692                    .0
693                    .clone()
694            };
695
696            let binds = bind_collector
697                .metadata
698                .into_iter()
699                .zip(bind_collector.binds)
700                .map(|(meta, bind)| ToSqlHelper(meta, bind))
701                .collect::<Vec<_>>();
702                callback(raw_connection, stmt.clone(), binds).await
703            };
704            let res = res.await;
705            let mut tm = tm.lock().await;
706            let r = update_transaction_manager_status(res, &mut tm);
707            instrumentation
708                .lock()
709                .unwrap_or_else(|p| p.into_inner())
710                .on_connection_event(InstrumentationEvent::finish_query(
711                    &StrQueryHelper::new(&sql),
712                    r.as_ref().err(),
713                ));
714
715            r
716        }
717        .boxed()
718    }
719
720    fn record_instrumentation(&self, event: InstrumentationEvent<'_>) {
721        self.instrumentation
722            .lock()
723            .unwrap_or_else(|p| p.into_inner())
724            .on_connection_event(event);
725    }
726
727    /// See Postgres documentation for SQL commands [NOTIFY][] and [LISTEN][]
728    ///
729    /// The returned stream yields all notifications received by the connection, not only notifications received
730    /// after calling the function. The returned stream will never close, so no notifications will just result
731    /// in a pending state.
732    ///
733    /// If there's no connection available to poll, the stream will yield no notifications and be pending forever.
734    /// This can happen if you created the [`AsyncPgConnection`] by the [`try_from`] constructor.
735    ///
736    /// [NOTIFY]: https://www.postgresql.org/docs/current/sql-notify.html
737    /// [LISTEN]: https://www.postgresql.org/docs/current/sql-listen.html
738    /// [`AsyncPgConnection`]: crate::pg::AsyncPgConnection
739    /// [`try_from`]: crate::pg::AsyncPgConnection::try_from
740    ///
741    /// ```rust
742    /// # include!("../doctest_setup.rs");
743    /// #
744    /// # #[tokio::main(flavor = "current_thread")]
745    /// # async fn main() {
746    /// #     run_test().await.unwrap();
747    /// # }
748    /// #
749    /// # async fn run_test() -> QueryResult<()> {
750    /// #     use diesel_async::RunQueryDsl;
751    /// #     use futures_util::StreamExt;
752    /// #     let conn = &mut connection_no_transaction().await;
753    /// // register the notifications channel we want to receive notifications for
754    /// diesel::sql_query("LISTEN example_channel").execute(conn).await?;
755    /// // send some notification (usually done from a different connection/thread/application)
756    /// diesel::sql_query("NOTIFY example_channel, 'additional data'").execute(conn).await?;
757    ///
758    /// let mut notifications = std::pin::pin!(conn.notifications_stream());
759    /// let mut notification = notifications.next().await.unwrap().unwrap();
760    ///
761    /// assert_eq!(notification.channel, "example_channel");
762    /// assert_eq!(notification.payload, "additional data");
763    /// println!("Notification received from process with id {}", notification.process_id);
764    /// # Ok(())
765    /// # }
766    /// ```
767    pub fn notifications_stream(
768        &mut self,
769    ) -> impl futures_core::Stream<Item = QueryResult<diesel::pg::PgNotification>> + '_ {
770        match &mut self.notification_rx {
771            None => Either::Left(futures_util::stream::pending()),
772            Some(rx) => Either::Right(futures_util::stream::unfold(rx, |rx| async {
773                rx.recv().await.map(move |item| (item, rx))
774            })),
775        }
776    }
777}
778
779struct BindData {
780    collect_bind_result: Result<(), Error>,
781    fake_oid_locations: Vec<(usize, usize)>,
782    generated_oids: GeneratedOidTypeMap,
783    bind_collector: RawBytesBindCollector<Pg>,
784}
785
786fn construct_bind_data(query: &dyn QueryFragment<diesel::pg::Pg>) -> BindData {
787    // we don't resolve custom types here yet, we do that later
788    // in the async block below as we might need to perform lookup
789    // queries for that.
790    //
791    // We apply this workaround to prevent requiring all the diesel
792    // serialization code to being async
793    //
794    // We give out constant fake oids here to optimize for the "happy" path
795    // without custom type lookup
796    let mut bind_collector_0 = RawBytesBindCollector::<diesel::pg::Pg>::new();
797    let mut metadata_lookup_0 = PgAsyncMetadataLookup {
798        custom_oid: false,
799        generated_oids: None,
800        oid_generator: Box::new(|_, _| (FAKE_OID, FAKE_OID)),
801    };
802    let collect_bind_result_0 =
803        query.collect_binds(&mut bind_collector_0, &mut metadata_lookup_0, &Pg);
804    // we have encountered a custom type oid, so we need to perform more work here.
805    // These oids can occur in two locations:
806    //
807    // * In the collected metadata -> relatively easy to resolve, just need to replace them below
808    // * As part of the serialized bind blob -> hard to replace
809    //
810    // To address the second case, we perform a second run of the bind collector
811    // with a different set of fake oids. Then we compare the output of the two runs
812    // and use that information to infer where to replace bytes in the serialized output
813    if metadata_lookup_0.custom_oid {
814        // we try to get the maximum oid we encountered here
815        // to be sure that we don't accidentally give out a fake oid below that collides with
816        // something
817        let mut max_oid = bind_collector_0
818            .metadata
819            .iter()
820            .flat_map(|t| {
821                [
822                    t.oid().unwrap_or_default(),
823                    t.array_oid().unwrap_or_default(),
824                ]
825            })
826            .max()
827            .unwrap_or_default();
828        let mut bind_collector_1 = RawBytesBindCollector::<diesel::pg::Pg>::new();
829        let mut metadata_lookup_1 = PgAsyncMetadataLookup {
830            custom_oid: false,
831            generated_oids: Some(HashMap::new()),
832            oid_generator: Box::new(move |_, _| {
833                max_oid += 2;
834                (max_oid, max_oid + 1)
835            }),
836        };
837        let collect_bind_result_1 =
838            query.collect_binds(&mut bind_collector_1, &mut metadata_lookup_1, &Pg);
839
840        assert_eq!(
841            bind_collector_0.binds.len(),
842            bind_collector_0.metadata.len()
843        );
844        let fake_oid_locations = std::iter::zip(
845            bind_collector_0
846                .binds
847                .iter()
848                .zip(&bind_collector_0.metadata),
849            &bind_collector_1.binds,
850        )
851        .enumerate()
852        .flat_map(|(bind_index, ((bytes_0, metadata_0), bytes_1))| {
853            // custom oids might appear in the serialized bind arguments for arrays or composite (record) types
854            // in both cases the relevant buffer is a custom type on it's own
855            // so we only need to check the cases that contain a fake OID on their own
856            let (bytes_0, bytes_1) = if matches!(metadata_0.oid(), Ok(FAKE_OID)) {
857                (
858                    bytes_0.as_deref().unwrap_or_default(),
859                    bytes_1.as_deref().unwrap_or_default(),
860                )
861            } else {
862                // for all other cases, just return an empty
863                // list to make the iteration below a no-op
864                // and prevent the need of boxing
865                (&[] as &[_], &[] as &[_])
866            };
867            let lookup_map = metadata_lookup_1
868                .generated_oids
869                .as_ref()
870                .map(|map| {
871                    map.values()
872                        .flat_map(|(oid, array_oid)| [*oid, *array_oid])
873                        .collect::<HashSet<_>>()
874                })
875                .unwrap_or_default();
876            std::iter::zip(
877                bytes_0.windows(std::mem::size_of_val(&FAKE_OID)),
878                bytes_1.windows(std::mem::size_of_val(&FAKE_OID)),
879            )
880            .enumerate()
881            .filter_map(move |(byte_index, (l, r))| {
882                // here we infer if some byte sequence is a fake oid
883                // We use the following conditions for that:
884                //
885                // * The first byte sequence matches the constant FAKE_OID
886                // * The second sequence does not match the constant FAKE_OID
887                // * The second sequence is contained in the set of generated oid,
888                //   otherwise we get false positives around the boundary
889                //   of a to be replaced byte sequence
890                let r_val = u32::from_be_bytes(r.try_into().expect("That's the right size"));
891                (l == FAKE_OID.to_be_bytes()
892                    && r != FAKE_OID.to_be_bytes()
893                    && lookup_map.contains(&r_val))
894                .then_some((bind_index, byte_index))
895            })
896        })
897        // Avoid storing the bind collectors in the returned Future
898        .collect::<Vec<_>>();
899        BindData {
900            collect_bind_result: collect_bind_result_0.and(collect_bind_result_1),
901            fake_oid_locations,
902            generated_oids: metadata_lookup_1.generated_oids,
903            bind_collector: bind_collector_1,
904        }
905    } else {
906        BindData {
907            collect_bind_result: collect_bind_result_0,
908            fake_oid_locations: Vec::new(),
909            generated_oids: None,
910            bind_collector: bind_collector_0,
911        }
912    }
913}
914
915type GeneratedOidTypeMap = Option<HashMap<(Option<String>, String), (u32, u32)>>;
916type OidGenerator = dyn FnMut(&str, Option<&str>) -> (u32, u32) + 'static;
917
918/// Collects types that need to be looked up, and causes fake OIDs to be written into the bind collector
919/// so they can be replaced with asynchronously fetched OIDs after the original query is dropped
920struct PgAsyncMetadataLookup {
921    custom_oid: bool,
922    generated_oids: GeneratedOidTypeMap,
923    oid_generator: Box<OidGenerator>,
924}
925
926impl PgMetadataLookup for PgAsyncMetadataLookup {
927    fn lookup_type(&mut self, type_name: &str, schema: Option<&str>) -> PgTypeMetadata {
928        self.custom_oid = true;
929
930        let oid = if let Some(map) = &mut self.generated_oids {
931            *map.entry((schema.map(ToOwned::to_owned), type_name.to_owned()))
932                .or_insert_with(|| (self.oid_generator)(type_name, schema))
933        } else {
934            (self.oid_generator)(type_name, schema)
935        };
936
937        PgTypeMetadata::from_result(Ok(oid))
938    }
939
940    fn as_any<'a>(&mut self) -> &mut (dyn std::any::Any + 'a)
941    where
942        Self: 'a,
943    {
944        self
945    }
946}
947
948impl AsyncMultiConnectionHelper for AsyncPgConnection {
949    fn to_any<'a>(
950        lookup: &mut <Self::Backend as TypeMetadata>::MetadataLookup,
951    ) -> &mut (dyn std::any::Any + 'a) {
952        lookup.as_any()
953    }
954
955    fn from_any(
956        lookup: &mut dyn std::any::Any,
957    ) -> Option<&mut <Self::Backend as diesel::sql_types::TypeMetadata>::MetadataLookup> {
958        lookup
959            .downcast_mut::<PgAsyncMetadataLookup>()
960            .map(|v| v as &mut dyn PgMetadataLookup)
961    }
962}
963
964async fn lookup_type(
965    schema: Option<String>,
966    type_name: String,
967    raw_connection: &tokio_postgres::Client,
968) -> QueryResult<(u32, u32)> {
969    let r = if let Some(schema) = schema.as_ref() {
970        raw_connection
971            .query_one(
972                "SELECT pg_type.oid, pg_type.typarray FROM pg_type \
973             INNER JOIN pg_namespace ON pg_type.typnamespace = pg_namespace.oid \
974             WHERE pg_type.typname = $1 AND pg_namespace.nspname = $2 \
975             LIMIT 1",
976                &[&type_name, schema],
977            )
978            .await
979            .map_err(ErrorHelper)?
980    } else {
981        raw_connection
982            .query_one(
983                "SELECT pg_type.oid, pg_type.typarray FROM pg_type \
984             WHERE pg_type.oid = quote_ident($1)::regtype::oid \
985             LIMIT 1",
986                &[&type_name],
987            )
988            .await
989            .map_err(ErrorHelper)?
990    };
991    Ok((r.get(0), r.get(1)))
992}
993
994fn unwrap_oids(metadata: &PgTypeMetadata) -> (u32, u32) {
995    let err_msg = "PgTypeMetadata is supposed to always be Ok here";
996    (
997        metadata.oid().expect(err_msg),
998        metadata.array_oid().expect(err_msg),
999    )
1000}
1001
1002fn replace_fake_oid(
1003    binds: &mut [Option<Vec<u8>>],
1004    real_oids: &HashMap<u32, u32>,
1005    bind_index: usize,
1006    byte_index: usize,
1007) -> Option<()> {
1008    let serialized_oid = binds
1009        .get_mut(bind_index)?
1010        .as_mut()?
1011        .get_mut(byte_index..)?
1012        .first_chunk_mut::<4>()?;
1013    *serialized_oid = real_oids
1014        .get(&u32::from_be_bytes(*serialized_oid))?
1015        .to_be_bytes();
1016    Some(())
1017}
1018
1019async fn drive_future<R>(
1020    connection_future: Option<broadcast::Receiver<Arc<tokio_postgres::Error>>>,
1021    client_future: impl Future<Output = Result<R, diesel::result::Error>>,
1022) -> Result<R, diesel::result::Error> {
1023    if let Some(mut connection_future) = connection_future {
1024        let client_future = std::pin::pin!(client_future);
1025        let connection_future = std::pin::pin!(connection_future.recv());
1026        match futures_util::future::select(client_future, connection_future).await {
1027            Either::Left((res, _)) => res,
1028            // we got an error from the background task
1029            // return it to the user
1030            Either::Right((Ok(e), _)) => Err(self::error_helper::from_tokio_postgres_error(e)),
1031            // seems like the background thread died for whatever reason
1032            Either::Right((Err(e), _)) => Err(diesel::result::Error::DatabaseError(
1033                DatabaseErrorKind::UnableToSendCommand,
1034                Box::new(e.to_string()),
1035            )),
1036        }
1037    } else {
1038        client_future.await
1039    }
1040}
1041
1042fn drive_connection<S>(
1043    mut conn: tokio_postgres::Connection<tokio_postgres::Socket, S>,
1044) -> (
1045    broadcast::Receiver<Arc<tokio_postgres::Error>>,
1046    mpsc::UnboundedReceiver<QueryResult<diesel::pg::PgNotification>>,
1047    oneshot::Sender<()>,
1048)
1049where
1050    S: tokio_postgres::tls::TlsStream + Unpin + Send + 'static,
1051{
1052    let (error_tx, error_rx) = tokio::sync::broadcast::channel(1);
1053    let (notification_tx, notification_rx) = tokio::sync::mpsc::unbounded_channel();
1054    let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
1055    let mut conn = futures_util::stream::poll_fn(move |cx| conn.poll_message(cx));
1056
1057    tokio::spawn(async move {
1058        loop {
1059            match futures_util::future::select(&mut shutdown_rx, conn.next()).await {
1060                Either::Left(_) | Either::Right((None, _)) => break,
1061                Either::Right((Some(Ok(tokio_postgres::AsyncMessage::Notification(notif))), _)) => {
1062                    let _: Result<_, _> = notification_tx.send(Ok(diesel::pg::PgNotification {
1063                        process_id: notif.process_id(),
1064                        channel: notif.channel().to_owned(),
1065                        payload: notif.payload().to_owned(),
1066                    }));
1067                }
1068                Either::Right((Some(Ok(_)), _)) => {}
1069                Either::Right((Some(Err(e)), _)) => {
1070                    let e = Arc::new(e);
1071                    let _: Result<_, _> = error_tx.send(e.clone());
1072                    let _: Result<_, _> =
1073                        notification_tx.send(Err(error_helper::from_tokio_postgres_error(e)));
1074                    break;
1075                }
1076            }
1077        }
1078    });
1079
1080    (error_rx, notification_rx, shutdown_tx)
1081}
1082
1083#[cfg(any(
1084    feature = "deadpool",
1085    feature = "bb8",
1086    feature = "mobc",
1087    feature = "r2d2"
1088))]
1089impl crate::pooled_connection::PoolableConnection for AsyncPgConnection {
1090    fn is_broken(&mut self) -> bool {
1091        use crate::TransactionManager;
1092
1093        Self::TransactionManager::is_broken_transaction_manager(self) || self.conn.is_closed()
1094    }
1095}
1096
1097impl QueryFragmentForCachedStatement<Pg> for QueryFragmentHelper {
1098    fn construct_sql(&self, _backend: &Pg) -> QueryResult<String> {
1099        Ok(self.sql.clone())
1100    }
1101
1102    fn is_safe_to_cache_prepared(&self, _backend: &Pg) -> QueryResult<bool> {
1103        Ok(self.safe_to_cache)
1104    }
1105
1106    fn parameter_count(&self, _backend: &Pg) -> QueryResult<usize> {
1107        Ok(self.bind_count)
1108    }
1109}
1110
1111impl PreparedStatement for StatementHelper<Statement> {
1112    fn parameter_count(&self) -> u64 {
1113        u64::try_from(self.0.params().len()).expect("usize is not larger than 64 bits")
1114    }
1115}
1116
1117#[cfg(test)]
1118mod tests {
1119    use super::*;
1120    use crate::run_query_dsl::RunQueryDsl;
1121    use diesel::IntoSql;
1122    use diesel::sql_types::Integer;
1123    use futures_util::future::try_join;
1124
1125    #[tokio::test]
1126    async fn pipelining() {
1127        let database_url = dotenvy::var("PG_DATABASE_URL")
1128            .or_else(|_| dotenvy::var("DATABASE_URL"))
1129            .expect("DATABASE_URL must be set in order to run tests");
1130
1131        let conn = crate::AsyncPgConnection::establish(&database_url)
1132            .await
1133            .unwrap();
1134
1135        let q1 = diesel::select(1_i32.into_sql::<Integer>());
1136        let q2 = diesel::select(2_i32.into_sql::<Integer>());
1137
1138        let f1 = q1.get_result::<i32>(&mut &conn);
1139        let f2 = q2.get_result::<i32>(&mut &conn);
1140
1141        let (r1, r2) = try_join(f1, f2).await.unwrap();
1142
1143        assert_eq!(r1, 1);
1144        assert_eq!(r2, 2);
1145    }
1146
1147    #[tokio::test]
1148    async fn pipelining_with_composed_futures() {
1149        let database_url = dotenvy::var("PG_DATABASE_URL")
1150            .or_else(|_| dotenvy::var("DATABASE_URL"))
1151            .expect("DATABASE_URL must be set in order to run tests");
1152
1153        let conn = crate::AsyncPgConnection::establish(&database_url)
1154            .await
1155            .unwrap();
1156
1157        async fn fn12(mut conn: &AsyncPgConnection) -> QueryResult<(i32, i32)> {
1158            let f1 = diesel::select(1_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
1159            let f2 = diesel::select(2_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
1160
1161            try_join(f1, f2).await
1162        }
1163
1164        async fn fn34(mut conn: &AsyncPgConnection) -> QueryResult<(i32, i32)> {
1165            let f3 = diesel::select(3_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
1166            let f4 = diesel::select(4_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
1167
1168            try_join(f3, f4).await
1169        }
1170
1171        let f12 = fn12(&conn);
1172        let f34 = fn34(&conn);
1173
1174        let ((r1, r2), (r3, r4)) = try_join(f12, f34).await.unwrap();
1175
1176        assert_eq!(r1, 1);
1177        assert_eq!(r2, 2);
1178        assert_eq!(r3, 3);
1179        assert_eq!(r4, 4);
1180    }
1181
1182    #[tokio::test]
1183    async fn pipelining_with_composed_futures_and_transaction() {
1184        let database_url = dotenvy::var("PG_DATABASE_URL")
1185            .or_else(|_| dotenvy::var("DATABASE_URL"))
1186            .expect("DATABASE_URL must be set in order to run tests");
1187
1188        let mut conn = crate::AsyncPgConnection::establish(&database_url)
1189            .await
1190            .unwrap();
1191
1192        async fn fn12(mut conn: &AsyncPgConnection) -> QueryResult<(i32, i32)> {
1193            let f1 = diesel::select(1_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
1194            let f2 = diesel::select(2_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
1195
1196            try_join(f1, f2).await
1197        }
1198
1199        async fn fn37(
1200            mut conn: &AsyncPgConnection,
1201        ) -> QueryResult<(usize, (Vec<i32>, (i32, (Vec<i32>, i32))))> {
1202            let f3 = diesel::select(0_i32.into_sql::<Integer>()).execute(&mut conn);
1203            let f4 = diesel::select(4_i32.into_sql::<Integer>()).load::<i32>(&mut conn);
1204            let f5 = diesel::select(5_i32.into_sql::<Integer>()).get_result::<i32>(&mut conn);
1205            let f6 = diesel::select(6_i32.into_sql::<Integer>()).get_results::<i32>(&mut conn);
1206            let f7 = diesel::select(7_i32.into_sql::<Integer>()).first::<i32>(&mut conn);
1207
1208            try_join(f3, try_join(f4, try_join(f5, try_join(f6, f7)))).await
1209        }
1210
1211        conn.transaction(async |conn| {
1212            let f12 = fn12(conn);
1213            let f37 = fn37(conn);
1214
1215            let ((r1, r2), (r3, (r4, (r5, (r6, r7))))) = try_join(f12, f37).await.unwrap();
1216
1217            assert_eq!(r1, 1);
1218            assert_eq!(r2, 2);
1219            assert_eq!(r3, 1);
1220            assert_eq!(r4, vec![4]);
1221            assert_eq!(r5, 5);
1222            assert_eq!(r6, vec![6]);
1223            assert_eq!(r7, 7);
1224
1225            fn12(conn).await?;
1226            fn37(conn).await?;
1227
1228            QueryResult::<_>::Ok(())
1229        })
1230        .await
1231        .unwrap();
1232    }
1233}