Skip to main content

chdb_rust/
arrow_insert.rs

1//! High-throughput inserts via Arrow C Data Interface registration.
2//!
3//! Wraps [`Connection::register_arrow_stream`](crate::connection::Connection::register_arrow_stream),
4//! [`Connection::register_arrow_array`](crate::connection::Connection::register_arrow_array),
5//! and `INSERT INTO … SELECT * FROM ArrowStream('name')`.
6
7#[cfg(not(direct_arrow_insert))]
8use std::sync::atomic::{AtomicU64, Ordering};
9
10use arrow::array::RecordBatchReader;
11use arrow::array::{RecordBatch, RecordBatchIterator, StructArray};
12use arrow::datatypes::SchemaRef;
13use arrow::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema};
14use arrow::ffi_stream::FFI_ArrowArrayStream;
15
16use crate::arrow_options::InsertOptions;
17use crate::arrow_stream::{arrow_stream_table_sql, ArrowArray, ArrowSchema, ArrowStream};
18use crate::connection::Connection;
19use crate::error::{Error, Result};
20use crate::format::OutputFormat;
21
22#[cfg(not(direct_arrow_insert))]
23static FALLBACK_STREAM_SEQ: AtomicU64 = AtomicU64::new(0);
24
25/// Owns Arrow C Data Interface structs for the lifetime of a registration or insert.
26struct OwnedArrowArray {
27    ffi_schema: FFI_ArrowSchema,
28    ffi_array: FFI_ArrowArray,
29}
30
31impl OwnedArrowArray {
32    fn from_batch(batch: RecordBatch) -> Result<Self> {
33        let struct_array = StructArray::from(batch);
34        let (ffi_array, ffi_schema) = to_ffi(&struct_array.into())
35            .map_err(|e| crate::error::Error::QueryError(e.to_string()))?;
36        Ok(Self {
37            ffi_schema,
38            ffi_array,
39        })
40    }
41
42    fn schema(&self) -> ArrowSchema {
43        unsafe { ArrowSchema::from_raw(&self.ffi_schema as *const _ as *mut _) }
44    }
45
46    fn array(&self) -> ArrowArray {
47        unsafe { ArrowArray::from_raw(&self.ffi_array as *const _ as *mut _) }
48    }
49}
50
51/// Backtick-quoted `(col1, col2, ...)` clause from an Arrow schema, or empty when
52/// the schema has no fields. Emitted on the `INSERT` so ClickHouse maps the Arrow
53/// columns to destination columns **by name**. Without it, `SELECT *` maps by
54/// position, which silently scatters values into the wrong columns whenever the
55/// table's physical column order differs from the batch's (e.g. after
56/// `ALTER TABLE ... ADD COLUMN`, which appends rather than preserving order).
57///
58/// The direct FFI path (`chdb_insert_arrow_array`) already matches by name via
59/// ClickHouse's Arrow input format; this keeps the SQL fallback consistent.
60fn column_list_clause(schema: &arrow::datatypes::Schema) -> String {
61    if schema.fields().is_empty() {
62        return String::new();
63    }
64    let cols = schema
65        .fields()
66        .iter()
67        .map(|f| format!("`{}`", f.name().replace('`', "``")))
68        .collect::<Vec<_>>()
69        .join(", ");
70    format!(" ({cols})")
71}
72
73fn insert_from_registered(
74    conn: &Connection,
75    dest_table: &str,
76    stream_name: &str,
77    columns: &str,
78    options: &InsertOptions,
79) -> Result<()> {
80    let mut sql = format!(
81        "INSERT INTO {dest_table}{columns} SELECT * FROM {}",
82        arrow_stream_table_sql(stream_name)
83    );
84    if let Some(settings) = options.settings_clause() {
85        sql.push_str(" SETTINGS ");
86        sql.push_str(&settings);
87    }
88    conn.query(&sql, OutputFormat::Null)?;
89    Ok(())
90}
91
92fn insert_via_registered_array(
93    conn: &Connection,
94    dest_table: &str,
95    stream_name: &str,
96    arrow_schema: &ArrowSchema,
97    arrow_array: &ArrowArray,
98    columns: &str,
99    options: &InsertOptions,
100) -> Result<()> {
101    conn.register_arrow_array(stream_name, arrow_schema, arrow_array)?;
102    let result = insert_from_registered(conn, dest_table, stream_name, columns, options);
103    finish_registered_insert(result, conn.unregister_arrow_table(stream_name))
104}
105
106fn finish_registered_insert(result: Result<()>, cleanup: Result<()>) -> Result<()> {
107    match (result, cleanup) {
108        (Ok(()), Ok(())) => Ok(()),
109        (Err(err), Ok(())) | (Ok(()), Err(err)) => Err(err),
110        (Err(err), Err(cleanup_err)) => Err(Error::QueryError(format!(
111            "{err}; Arrow source cleanup also failed: {cleanup_err}"
112        ))),
113    }
114}
115
116/// Insert a single [`RecordBatch`] into `dest_table` via `register_arrow_array`.
117///
118/// `stream_name` is a temporary registration label (not a MergeTree table name).
119/// `dest_table` must be a bare ClickHouse table identifier (e.g. `metrics`);
120/// dotted names (`db.table`) and reserved identifiers are not quoted automatically.
121/// Uses the fastest one-shot Arrow path for a single in-memory batch.
122///
123/// # Examples
124///
125/// ```no_run
126/// use std::sync::Arc;
127/// use arrow::array::{Int64Array, RecordBatch};
128/// use arrow::datatypes::{DataType, Field, Schema};
129/// use chdb_rust::arrow_insert::insert_record_batch;
130/// use chdb_rust::InsertOptions;
131/// use chdb_rust::connection::Connection;
132///
133/// let conn = Connection::open_in_memory()?;
134/// conn.query(
135///     "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
136///     chdb_rust::format::OutputFormat::TabSeparated,
137/// )?;
138///
139/// let batch = RecordBatch::try_new(
140///     Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])),
141///     vec![Arc::new(Int64Array::from(vec![1i64, 2, 3]))],
142/// )
143/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?;
144///
145/// insert_record_batch(&conn, "metrics", "flush_1", batch, InsertOptions::default_bulk())?;
146/// # Ok::<(), chdb_rust::error::Error>(())
147/// ```
148///
149/// # Errors
150///
151/// Returns an error if registration, the insert query, or unregistration fails.
152pub fn insert_record_batch(
153    conn: &Connection,
154    dest_table: &str,
155    stream_name: &str,
156    batch: RecordBatch,
157    options: InsertOptions,
158) -> Result<()> {
159    let columns = column_list_clause(batch.schema().as_ref());
160    let handles = OwnedArrowArray::from_batch(batch)?;
161    insert_via_registered_array(
162        conn,
163        dest_table,
164        stream_name,
165        &handles.schema(),
166        &handles.array(),
167        &columns,
168        &options,
169    )
170}
171
172/// Insert a single [`RecordBatch`] using the direct `chdb_insert_arrow_array` FFI when
173/// available, falling back to register + SQL + unregister otherwise.
174///
175/// Unlike [`insert_record_batch`], no temporary `stream_name` is required on the direct path.
176/// This is the recommended entry point for single-batch ingest.
177///
178/// # Examples
179///
180/// ```no_run
181/// use std::sync::Arc;
182/// use arrow::array::{Int64Array, RecordBatch};
183/// use arrow::datatypes::{DataType, Field, Schema};
184/// use chdb_rust::arrow_insert::insert_record_batch_direct;
185/// use chdb_rust::InsertOptions;
186/// use chdb_rust::connection::Connection;
187///
188/// let conn = Connection::open_in_memory()?;
189/// conn.query(
190///     "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
191///     chdb_rust::format::OutputFormat::TabSeparated,
192/// )?;
193///
194/// let batch = RecordBatch::try_new(
195///     Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])),
196///     vec![Arc::new(Int64Array::from(vec![1i64, 2, 3]))],
197/// )
198/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?;
199///
200/// insert_record_batch_direct(&conn, "metrics", batch, InsertOptions::default_bulk())?;
201/// # Ok::<(), chdb_rust::error::Error>(())
202/// ```
203pub fn insert_record_batch_direct(
204    conn: &Connection,
205    dest_table: &str,
206    batch: RecordBatch,
207    options: InsertOptions,
208) -> Result<()> {
209    let columns = column_list_clause(batch.schema().as_ref());
210    let handles = OwnedArrowArray::from_batch(batch)?;
211
212    #[cfg(direct_arrow_insert)]
213    {
214        let _ = &columns;
215        conn.insert_arrow_array(dest_table, &handles.schema(), &handles.array(), &options)
216    }
217
218    #[cfg(not(direct_arrow_insert))]
219    {
220        let stream_name = format!(
221            "__rust_direct_{}",
222            FALLBACK_STREAM_SEQ.fetch_add(1, Ordering::Relaxed)
223        );
224        insert_via_registered_array(
225            conn,
226            dest_table,
227            &stream_name,
228            &handles.schema(),
229            &handles.array(),
230            &columns,
231            &options,
232        )
233    }
234}
235
236/// Insert multiple [`RecordBatch`] values via `register_arrow_stream`.
237///
238/// `batches` must share the same schema as `schema`. Prefer this when a flush
239/// spans more than one batch. See [`insert_record_batch`] for `dest_table` constraints.
240///
241/// # Examples
242///
243/// ```no_run
244/// use std::sync::Arc;
245/// use arrow::array::{Int64Array, RecordBatch};
246/// use arrow::datatypes::{DataType, Field, Schema};
247/// use chdb_rust::arrow_insert::insert_record_batches;
248/// use chdb_rust::InsertOptions;
249/// use chdb_rust::connection::Connection;
250///
251/// let conn = Connection::open_in_memory()?;
252/// conn.query(
253///     "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
254///     chdb_rust::format::OutputFormat::TabSeparated,
255/// )?;
256///
257/// let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
258/// let batches = vec![
259///     RecordBatch::try_new(
260///         Arc::clone(&schema),
261///         vec![Arc::new(Int64Array::from(vec![1i64, 2]))],
262///     )
263///     .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?,
264///     RecordBatch::try_new(
265///         Arc::clone(&schema),
266///         vec![Arc::new(Int64Array::from(vec![3i64, 4]))],
267///     )
268///     .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?,
269/// ];
270///
271/// insert_record_batches(
272///     &conn,
273///     "metrics",
274///     "flush_multi",
275///     schema,
276///     batches,
277///     InsertOptions::default_bulk(),
278/// )?;
279/// # Ok::<(), chdb_rust::error::Error>(())
280/// ```
281pub fn insert_record_batches(
282    conn: &Connection,
283    dest_table: &str,
284    stream_name: &str,
285    schema: SchemaRef,
286    batches: Vec<RecordBatch>,
287    options: InsertOptions,
288) -> Result<()> {
289    let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
290    insert_record_batch_reader(conn, dest_table, stream_name, Box::new(reader), options)
291}
292
293/// Insert from any [`RecordBatchReader`] via `register_arrow_stream`, or direct FFI when available.
294///
295/// See [`insert_record_batch`] for `dest_table` constraints.
296///
297/// # Examples
298///
299/// ```no_run
300/// use std::sync::Arc;
301/// use arrow::array::{Int64Array, RecordBatch, RecordBatchIterator};
302/// use arrow::datatypes::{DataType, Field, Schema};
303/// use chdb_rust::arrow_insert::insert_record_batch_reader;
304/// use chdb_rust::InsertOptions;
305/// use chdb_rust::connection::Connection;
306///
307/// let conn = Connection::open_in_memory()?;
308/// conn.query(
309///     "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
310///     chdb_rust::format::OutputFormat::TabSeparated,
311/// )?;
312///
313/// let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
314/// let batch = RecordBatch::try_new(
315///     Arc::clone(&schema),
316///     vec![Arc::new(Int64Array::from(vec![1i64, 2]))],
317/// )
318/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?;
319/// let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
320///
321/// insert_record_batch_reader(
322///     &conn,
323///     "metrics",
324///     "flush_reader",
325///     Box::new(reader),
326///     InsertOptions::default_bulk(),
327/// )?;
328/// # Ok::<(), chdb_rust::error::Error>(())
329/// ```
330pub fn insert_record_batch_reader(
331    conn: &Connection,
332    dest_table: &str,
333    stream_name: &str,
334    reader: Box<dyn RecordBatchReader + Send>,
335    options: InsertOptions,
336) -> Result<()> {
337    let columns = column_list_clause(reader.schema().as_ref());
338    let mut ffi_stream = FFI_ArrowArrayStream::new(reader);
339    let arrow_stream = unsafe { ArrowStream::from_raw(&mut ffi_stream) };
340
341    #[cfg(direct_arrow_insert)]
342    {
343        let _ = (stream_name, &columns);
344        conn.insert_arrow_stream(dest_table, &arrow_stream, &options)
345    }
346
347    #[cfg(not(direct_arrow_insert))]
348    {
349        conn.register_arrow_stream(stream_name, &arrow_stream)?;
350        let result = insert_from_registered(conn, dest_table, stream_name, &columns, &options);
351        finish_registered_insert(result, conn.unregister_arrow_table(stream_name))
352    }
353}