arrow_odbc/
odbc_writer.rs

1use std::{borrow::Cow, cmp::min, sync::Arc};
2
3use thiserror::Error;
4
5use arrow::{
6    array::Array,
7    datatypes::{
8        DataType, Date32Type, Date64Type, Field, Float16Type, Float32Type, Float64Type, Int8Type,
9        Int16Type, Int32Type, Int64Type, Schema, Time32MillisecondType, Time32SecondType,
10        Time64MicrosecondType, Time64NanosecondType, TimeUnit, UInt8Type,
11    },
12    error::ArrowError,
13    record_batch::{RecordBatch, RecordBatchReader},
14};
15use odbc_api::{
16    ColumnarBulkInserter, Connection, ConnectionTransitions, Prepared,
17    buffers::{AnyBuffer, AnySliceMut, BufferDesc},
18    handles::{AsStatementRef, StatementConnection, StatementImpl, StatementParent},
19};
20
21use crate::{
22    date_time::{NullableTimeAsText, epoch_to_date, sec_since_midnight_to_time},
23    decimal::{NullableDecimal128AsText, NullableDecimal256AsText},
24    odbc_writer::timestamp::insert_timestamp_strategy,
25};
26
27use self::{
28    binary::VariadicBinary,
29    boolean::boolean_to_bit,
30    map_arrow_to_odbc::MapArrowToOdbc,
31    text::{LargeUtf8ToNativeText, Utf8ToNativeText},
32};
33
34mod binary;
35mod boolean;
36mod map_arrow_to_odbc;
37mod text;
38mod timestamp;
39
40/// Fastest and most convinient way to stream the contents of arrow record batches into a database
41/// table. For usecase there you want to insert repeatedly into the same table from different
42/// streams it is more efficient to create an instance of [`self::OdbcWriter`] and reuse it.
43///
44/// **Note:**
45///
46/// If table or column names are derived from user input, be sure to sanatize the input in order to
47/// prevent SQL injection attacks.
48pub fn insert_into_table(
49    connection: &Connection,
50    batches: &mut impl RecordBatchReader,
51    table_name: &str,
52    batch_size: usize,
53) -> Result<(), WriterError> {
54    let schema = batches.schema();
55    let mut inserter =
56        OdbcWriter::with_connection(connection, schema.as_ref(), table_name, batch_size)?;
57    inserter.write_all(batches)
58}
59
60/// Generates an insert statement using the table and column names.
61///
62/// `INSERT INTO <table> (<column_names 0>, <column_names 1>, ...) VALUES (?, ?, ...)`
63fn insert_statement_text(table: &str, column_names: &[&'_ str]) -> String {
64    // Generate statement text from table name and headline
65    let column_names = column_names
66        .iter()
67        .map(|cn| quote_column_name(cn))
68        .collect::<Vec<_>>();
69    let columns = column_names.join(", ");
70    let values = column_names
71        .iter()
72        .map(|_| "?")
73        .collect::<Vec<_>>()
74        .join(", ");
75    // Do not finish the statement with a semicolon. There is anecodtical evidence of IBM db2 not
76    // allowing the command, because it expects now multiple statements.
77    // See: <https://github.com/pacman82/arrow-odbc/issues/63>
78    format!("INSERT INTO {table} ({columns}) VALUES ({values})")
79}
80
81/// Wraps column name in quotes, if need be
82fn quote_column_name(column_name: &str) -> Cow<'_, str> {
83    if column_name.contains(|c| !valid_in_column_name(c)) {
84        Cow::Owned(format!("\"{column_name}\""))
85    } else {
86        Cow::Borrowed(column_name)
87    }
88}
89
90/// Check if this character is allowed in an unquoted column name
91fn valid_in_column_name(c: char) -> bool {
92    // See:
93    // <https://stackoverflow.com/questions/4200351/what-characters-are-valid-in-an-sql-server-database-name>
94    c.is_alphanumeric() || c == '@' || c == '$' || c == '#' || c == '_'
95}
96
97/// Creates an SQL insert statement from an arrow schema. The resulting statement will have one
98/// placeholer (`?`) for each column in the statement.
99///
100/// **Note:**
101///
102/// If the column name contains any character which would make it not a valid qualifier for transact
103/// SQL it will be wrapped in double quotes (`"`) within the insert schema. Valid names consist of
104/// alpha numeric characters, `@`, `$`, `#` and `_`.
105///
106/// # Example
107///
108/// ```
109/// use arrow_odbc::{
110///     insert_statement_from_schema,
111///     arrow::datatypes::{Field, DataType, Schema},
112/// };
113///
114/// let field_a = Field::new("a", DataType::Int64, false);
115/// let field_b = Field::new("b", DataType::Boolean, false);
116///
117/// let schema = Schema::new(vec![field_a, field_b]);
118/// let sql = insert_statement_from_schema(&schema, "MyTable");
119///
120/// assert_eq!("INSERT INTO MyTable (a, b) VALUES (?, ?)", sql)
121/// ```
122///
123/// This function is automatically invoked by [`crate::OdbcWriter::with_connection`].
124pub fn insert_statement_from_schema(schema: &Schema, table_name: &str) -> String {
125    let fields = schema.fields();
126    let num_columns = fields.len();
127    let column_names: Vec<_> = (0..num_columns)
128        .map(|i| fields[i].name().as_str())
129        .collect();
130    insert_statement_text(table_name, &column_names)
131}
132
133/// Emitted writing values from arror arrays into a table on the database
134#[derive(Debug, Error)]
135pub enum WriterError {
136    #[error("Failure to bind the array parameter buffers to the statement.\n{0}")]
137    BindParameterBuffers(#[source] odbc_api::Error),
138    #[error("Failure to execute the sql statement, sending the data to the database.\n{0}")]
139    ExecuteStatment(#[source] odbc_api::Error),
140    #[error("An error occured rebinding a parameter buffer to the sql statement.\n{0}")]
141    RebindBuffer(#[source] odbc_api::Error),
142    #[error("The arrow data type {0} is not supported for insertion.")]
143    UnsupportedArrowDataType(DataType),
144    #[error("An error occured extracting a record batch from an error reader.\n{0}")]
145    ReadingRecordBatch(#[source] ArrowError),
146    #[error("Unable to parse '{time_zone}' into a valid IANA time zone.")]
147    InvalidTimeZone { time_zone: Arc<str> },
148    #[error("An error occurred preparing SQL statement. SQL:\n{sql}\n{source}")]
149    PreparingInsertStatement {
150        #[source]
151        source: odbc_api::Error,
152        sql: String,
153    },
154}
155
156/// Inserts batches from an [`arrow::record_batch::RecordBatchReader`] into a database.
157pub struct OdbcWriter<S> {
158    /// Prepared statement with bound array parameter buffers. Data is copied into these buffers
159    /// until they are full. Then we execute the statement. This is repeated until we run out of
160    /// data.
161    inserter: ColumnarBulkInserter<S, AnyBuffer>,
162    /// For each field in the arrow schema we decide on which buffer to use to send the parameters
163    /// to the database, and need to remember how to copy the data from an arrow array to an odbc
164    /// mutable buffer slice for any column.
165    strategies: Vec<Box<dyn WriteStrategy>>,
166}
167
168impl<S> OdbcWriter<S>
169where
170    S: AsStatementRef,
171{
172    /// Construct a new ODBC writer using an alredy existing prepared statement. Usually you want to
173    /// call a higher level constructor like [`Self::with_connection`]. Yet, this constructor is
174    /// useful in two scenarios.
175    ///
176    /// 1. The prepared statement is already constructed and you do not want to spend the time to
177    ///    prepare it again.
178    /// 2. You want to use the arrow arrays as arrar parameters for a statement, but that statement
179    ///    is not necessarily an INSERT statement with a simple 1to1 mapping of columns between
180    ///    table and arrow schema.
181    ///
182    /// # Parameters
183    ///
184    /// * `row_capacity`: The amount of rows send to the database in each chunk. With the exception
185    ///   of the last chunk, which may be smaller.
186    /// * `schema`: Schema needs to have one column for each positional parameter of the statement
187    ///   and match the data which will be supplied to the instance later. Otherwise your code will
188    ///   panic.
189    /// * `statement`: A prepared statement whose SQL text representation contains one placeholder
190    ///   for each column. The order of the placeholers must correspond to the orders of the columns
191    ///   in the `schema`.
192    pub fn new(
193        row_capacity: usize,
194        schema: &Schema,
195        statement: Prepared<S>,
196    ) -> Result<Self, WriterError> {
197        let strategies: Vec<_> = schema
198            .fields()
199            .iter()
200            .map(|field| field_to_write_strategy(field.as_ref()))
201            .collect::<Result<_, _>>()?;
202        let descriptions = strategies.iter().map(|cws| cws.buffer_desc());
203        let inserter = statement
204            .into_column_inserter(row_capacity, descriptions)
205            .map_err(WriterError::BindParameterBuffers)?;
206
207        Ok(Self {
208            inserter,
209            strategies,
210        })
211    }
212
213    /// Consumes all the batches in the record batch reader and sends them chunk by chunk to the
214    /// database.
215    pub fn write_all(
216        &mut self,
217        reader: impl Iterator<Item = Result<RecordBatch, ArrowError>>,
218    ) -> Result<(), WriterError> {
219        for result in reader {
220            let record_batch = result.map_err(WriterError::ReadingRecordBatch)?;
221            self.write_batch(&record_batch)?;
222        }
223        self.flush()?;
224        Ok(())
225    }
226
227    /// Consumes a single batch and sends it chunk by chunk to the database. The last batch may not
228    /// be consumed until [`Self::flush`] is called.
229    pub fn write_batch(&mut self, record_batch: &RecordBatch) -> Result<(), WriterError> {
230        let capacity = self.inserter.capacity();
231        let mut remanining_rows = record_batch.num_rows();
232        // The record batch may contain more rows than the capacity of our writer can hold. So we
233        // need to be able to fill the buffers multiple times and send them to the database in
234        // between.
235        while remanining_rows != 0 {
236            let chunk_size = min(capacity - self.inserter.num_rows(), remanining_rows);
237            let param_offset = self.inserter.num_rows();
238            self.inserter.set_num_rows(param_offset + chunk_size);
239            let chunk = record_batch.slice(record_batch.num_rows() - remanining_rows, chunk_size);
240            for (index, (array, strategy)) in chunk
241                .columns()
242                .iter()
243                .zip(self.strategies.iter())
244                .enumerate()
245            {
246                strategy.write_rows(param_offset, self.inserter.column_mut(index), array)?
247            }
248
249            // If we used up all capacity we send the parameters to the database and reset the
250            // parameter buffers.
251            if self.inserter.num_rows() == capacity {
252                self.flush()?;
253            }
254            remanining_rows -= chunk_size;
255        }
256
257        Ok(())
258    }
259
260    /// The number of row in an individual record batch must not necessarily match the capacity of
261    /// the buffers owned by this writer. Therfore sometimes records are not send to the database
262    /// immediatly but rather we wait for the buffers to be filled then reading the next batch. Once
263    /// we reach the last batch however, there is no "next batch" anymore. In that case we call this
264    /// method in order to send the remainder of the records to the database as well.
265    pub fn flush(&mut self) -> Result<(), WriterError> {
266        self.inserter
267            .execute()
268            .map_err(WriterError::ExecuteStatment)?;
269        self.inserter.clear();
270        Ok(())
271    }
272}
273
274impl<C> OdbcWriter<StatementConnection<C>>
275where
276    C: StatementParent,
277{
278    /// A writer which takes ownership of the connection and inserts the given schema into a table
279    /// with matching column names.
280    ///
281    /// **Note:**
282    ///
283    /// If the column name contains any character which would make it not a valid qualifier for transact
284    /// SQL it will be wrapped in double quotes (`"`) within the insert schema. Valid names consist of
285    /// alpha numeric characters, `@`, `$`, `#` and `_`.
286    pub fn from_connection<C2>(
287        connection: C2,
288        schema: &Schema,
289        table_name: &str,
290        row_capacity: usize,
291    ) -> Result<Self, WriterError>
292    where
293        C2: ConnectionTransitions<StatementParent = C>,
294    {
295        let sql = insert_statement_from_schema(schema, table_name);
296        let statement = connection
297            .into_prepared(&sql)
298            .map_err(|source| WriterError::PreparingInsertStatement { source, sql })?;
299        Self::new(row_capacity, schema, statement)
300    }
301}
302
303impl<'o> OdbcWriter<StatementImpl<'o>> {
304    /// A writer which borrows the connection and inserts the given schema into a table with
305    /// matching column names.
306    ///
307    /// **Note:**
308    ///
309    /// If the column name contains any character which would make it not a valid qualifier for transact
310    /// SQL it will be wrapped in double quotes (`"`) within the insert schema. Valid names consist of
311    /// alpha numeric characters, `@`, `$`, `#` and `_`.
312    pub fn with_connection(
313        connection: &'o Connection<'o>,
314        schema: &Schema,
315        table_name: &str,
316        row_capacity: usize,
317    ) -> Result<Self, WriterError> {
318        let sql = insert_statement_from_schema(schema, table_name);
319        let statement = connection
320            .prepare(&sql)
321            .map_err(|source| WriterError::PreparingInsertStatement { source, sql })?;
322        Self::new(row_capacity, schema, statement)
323    }
324}
325
326pub trait WriteStrategy {
327    /// Describe the buffer used to hold the array parameters for the column
328    fn buffer_desc(&self) -> BufferDesc;
329
330    /// # Parameters
331    ///
332    /// * `param_offset`: Start writing parameters at that position. Number of rows in the parameter
333    ///   buffer before inserting the current chunk.
334    /// * `column_buf`: Buffer to write the data into
335    /// * `array`: Buffer to read the data from
336    fn write_rows(
337        &self,
338        param_offset: usize,
339        column_buf: AnySliceMut<'_>,
340        array: &dyn Array,
341    ) -> Result<(), WriterError>;
342}
343
344fn field_to_write_strategy(field: &Field) -> Result<Box<dyn WriteStrategy>, WriterError> {
345    let is_nullable = field.is_nullable();
346    let strategy = match field.data_type() {
347        DataType::Utf8 => Box::new(Utf8ToNativeText {}),
348        DataType::Boolean => boolean_to_bit(is_nullable),
349        DataType::LargeUtf8 => Box::new(LargeUtf8ToNativeText {}),
350        DataType::Int8 => Int8Type::identical(is_nullable),
351        DataType::Int16 => Int16Type::identical(is_nullable),
352        DataType::Int32 => Int32Type::identical(is_nullable),
353        DataType::Int64 => Int64Type::identical(is_nullable),
354        DataType::UInt8 => UInt8Type::identical(is_nullable),
355        DataType::Float16 => Float16Type::map_with(is_nullable, |half| half.to_f32()),
356        DataType::Float32 => Float32Type::identical(is_nullable),
357        DataType::Float64 => Float64Type::identical(is_nullable),
358        DataType::Timestamp(time_unit, time_zone) => {
359            insert_timestamp_strategy(is_nullable, &time_unit, time_zone.clone())?
360        }
361        DataType::Date32 => Date32Type::map_with(is_nullable, epoch_to_date),
362        DataType::Date64 => Date64Type::map_with(is_nullable, |days_since_epoch| {
363            epoch_to_date(days_since_epoch.try_into().unwrap())
364        }),
365        DataType::Time32(TimeUnit::Second) => {
366            Time32SecondType::map_with(is_nullable, sec_since_midnight_to_time)
367        }
368        DataType::Time32(TimeUnit::Millisecond) => {
369            Box::new(NullableTimeAsText::<Time32MillisecondType>::new())
370        }
371        DataType::Time64(TimeUnit::Microsecond) => {
372            Box::new(NullableTimeAsText::<Time64MicrosecondType>::new())
373        }
374        DataType::Time64(TimeUnit::Nanosecond) => {
375            Box::new(NullableTimeAsText::<Time64NanosecondType>::new())
376        }
377        DataType::Binary => Box::new(VariadicBinary::new(1)),
378        DataType::FixedSizeBinary(length) => {
379            Box::new(VariadicBinary::new((*length).try_into().unwrap()))
380        }
381        DataType::Decimal128(precision, scale) => {
382            Box::new(NullableDecimal128AsText::new(*precision, *scale))
383        }
384        DataType::Decimal256(precision, scale) => {
385            Box::new(NullableDecimal256AsText::new(*precision, *scale))
386        }
387        unsupported => return Err(WriterError::UnsupportedArrowDataType(unsupported.clone())),
388    };
389    Ok(strategy)
390}
arrow_odbc/odbc_writer.rs

arrow_odbc/
odbc_writer.rs
