Skip to main content

rok_db_core/
copy.rs

1//! Bulk loading with PostgreSQL `COPY … FROM STDIN (FORMAT binary)`.
2
3use sqlx::postgres::PgConnection;
4
5use crate::{Db, Error, Model, Result, Tx};
6
7/// Where [`Model::copy_in`] runs: a pool (one connection is borrowed), a
8/// transaction or a plain connection.
9#[derive(Debug)]
10pub enum CopyTarget<'a> {
11    /// A connection borrowed from the pool.
12    Db(&'a Db),
13    /// An open transaction.
14    Tx(&'a mut Tx),
15    /// A plain connection.
16    Connection(&'a mut PgConnection),
17}
18
19impl<'a> From<&'a Db> for CopyTarget<'a> {
20    fn from(db: &'a Db) -> Self {
21        CopyTarget::Db(db)
22    }
23}
24
25impl<'a> From<&'a mut Tx> for CopyTarget<'a> {
26    fn from(tx: &'a mut Tx) -> Self {
27        CopyTarget::Tx(tx)
28    }
29}
30
31impl<'a> From<&'a mut PgConnection> for CopyTarget<'a> {
32    fn from(conn: &'a mut PgConnection) -> Self {
33        CopyTarget::Connection(conn)
34    }
35}
36
37const SIGNATURE: &[u8] = b"PGCOPY\n\xff\r\n\0";
38/// Flush to the server roughly every megabyte.
39const CHUNK: usize = 1 << 20;
40
41pub(crate) async fn copy_in<M: Model>(target: CopyTarget<'_>, records: &[M]) -> Result<u64> {
42    if records.is_empty() {
43        return Ok(0);
44    }
45    for record in records {
46        crate::model::pre_insert(record)?;
47    }
48    let columns: Vec<&'static str> = M::COLUMNS
49        .iter()
50        .copied()
51        .filter(|c| !M::GENERATED.contains(c))
52        .collect();
53    let mut statement = String::from("COPY ");
54    crate::sql::push_ident(&mut statement, M::TABLE);
55    statement.push_str(" (");
56    for (i, c) in columns.iter().enumerate() {
57        if i > 0 {
58            statement.push_str(", ");
59        }
60        crate::sql::push_ident(&mut statement, c);
61    }
62    statement.push_str(") FROM STDIN (FORMAT binary)");
63
64    match target {
65        CopyTarget::Db(db) => {
66            let mut conn = db.pool().acquire().await?;
67            let rows = run(&mut conn, &statement, &columns, records).await?;
68            if let Some(cache) = db.cache() {
69                cache.invalidate(M::TABLE);
70            }
71            Ok(rows)
72        }
73        CopyTarget::Tx(tx) => {
74            let rows = run(tx, &statement, &columns, records).await?;
75            tx.note_write(M::TABLE);
76            Ok(rows)
77        }
78        CopyTarget::Connection(conn) => run(conn, &statement, &columns, records).await,
79    }
80}
81
82async fn run<M: Model>(
83    conn: &mut PgConnection,
84    statement: &str,
85    columns: &[&'static str],
86    records: &[M],
87) -> Result<u64> {
88    let start = std::time::Instant::now();
89    let mut copy = conn.copy_in_raw(statement).await?;
90    let mut buf = Vec::with_capacity(CHUNK + 4096);
91    buf.extend_from_slice(SIGNATURE);
92    buf.extend_from_slice(&0_i32.to_be_bytes()); // flags
93    buf.extend_from_slice(&0_i32.to_be_bytes()); // header extension length
94
95    let field_count = i16::try_from(columns.len())
96        .map_err(|_| Error::InvalidQuery("too many columns for COPY".into()))?;
97    for record in records {
98        buf.extend_from_slice(&field_count.to_be_bytes());
99        let values = record.values();
100        for (column, value) in values.iter().filter(|(c, _)| columns.contains(c)) {
101            let tenant = crate::model::tenant_override::<M>(column);
102            let value = tenant.as_ref().unwrap_or(value);
103            match value.encode_binary() {
104                Ok(Some(bytes)) => {
105                    let len = i32::try_from(bytes.len()).map_err(|_| {
106                        Error::InvalidQuery(format!("value of `{column}` is too large for COPY"))
107                    })?;
108                    buf.extend_from_slice(&len.to_be_bytes());
109                    buf.extend_from_slice(&bytes);
110                }
111                Ok(None) => buf.extend_from_slice(&(-1_i32).to_be_bytes()),
112                Err(e) => {
113                    let _ = copy.abort("rok-db: failed to encode a value").await;
114                    return Err(e);
115                }
116            }
117        }
118        if buf.len() >= CHUNK {
119            copy.send(std::mem::take(&mut buf)).await?;
120            buf.reserve(CHUNK + 4096);
121        }
122    }
123    buf.extend_from_slice(&(-1_i16).to_be_bytes()); // trailer
124    copy.send(buf).await?;
125    let rows = copy.finish().await?;
126    crate::metrics::query(statement, start.elapsed(), Some(rows), false);
127    tracing::debug!(
128        target: "rok_db::query",
129        sql = statement,
130        rows,
131        elapsed_ms = start.elapsed().as_secs_f64() * 1000.0,
132        "copy"
133    );
134    Ok(rows)
135}