chdb_rust/arrow_insert.rs
1//! High-throughput inserts via Arrow C Data Interface registration.
2//!
3//! Wraps [`Connection::register_arrow_stream`](crate::connection::Connection::register_arrow_stream),
4//! [`Connection::register_arrow_array`](crate::connection::Connection::register_arrow_array),
5//! and `INSERT INTO … SELECT * FROM ArrowStream('name')`.
6
7#[cfg(not(direct_arrow_insert))]
8use std::sync::atomic::{AtomicU64, Ordering};
9
10use arrow::array::RecordBatchReader;
11use arrow::array::{RecordBatch, RecordBatchIterator, StructArray};
12use arrow::datatypes::SchemaRef;
13use arrow::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema};
14use arrow::ffi_stream::FFI_ArrowArrayStream;
15
16use crate::arrow_options::InsertOptions;
17use crate::arrow_stream::{arrow_stream_table_sql, ArrowArray, ArrowSchema, ArrowStream};
18use crate::connection::Connection;
19use crate::error::{Error, Result};
20use crate::format::OutputFormat;
21
22#[cfg(not(direct_arrow_insert))]
23static FALLBACK_STREAM_SEQ: AtomicU64 = AtomicU64::new(0);
24
25/// Owns Arrow C Data Interface structs for the lifetime of a registration or insert.
26struct OwnedArrowArray {
27 ffi_schema: FFI_ArrowSchema,
28 ffi_array: FFI_ArrowArray,
29}
30
31impl OwnedArrowArray {
32 fn from_batch(batch: RecordBatch) -> Result<Self> {
33 let struct_array = StructArray::from(batch);
34 let (ffi_array, ffi_schema) = to_ffi(&struct_array.into())
35 .map_err(|e| crate::error::Error::QueryError(e.to_string()))?;
36 Ok(Self {
37 ffi_schema,
38 ffi_array,
39 })
40 }
41
42 fn schema(&self) -> ArrowSchema {
43 unsafe { ArrowSchema::from_raw(&self.ffi_schema as *const _ as *mut _) }
44 }
45
46 fn array(&self) -> ArrowArray {
47 unsafe { ArrowArray::from_raw(&self.ffi_array as *const _ as *mut _) }
48 }
49}
50
51/// Backtick-quoted `(col1, col2, ...)` clause from an Arrow schema, or empty when
52/// the schema has no fields. Emitted on the `INSERT` so ClickHouse maps the Arrow
53/// columns to destination columns **by name**. Without it, `SELECT *` maps by
54/// position, which silently scatters values into the wrong columns whenever the
55/// table's physical column order differs from the batch's (e.g. after
56/// `ALTER TABLE ... ADD COLUMN`, which appends rather than preserving order).
57///
58/// The direct FFI path (`chdb_insert_arrow_array`) already matches by name via
59/// ClickHouse's Arrow input format; this keeps the SQL fallback consistent.
60fn column_list_clause(schema: &arrow::datatypes::Schema) -> String {
61 if schema.fields().is_empty() {
62 return String::new();
63 }
64 let cols = schema
65 .fields()
66 .iter()
67 .map(|f| format!("`{}`", f.name().replace('`', "``")))
68 .collect::<Vec<_>>()
69 .join(", ");
70 format!(" ({cols})")
71}
72
73fn insert_from_registered(
74 conn: &Connection,
75 dest_table: &str,
76 stream_name: &str,
77 columns: &str,
78 options: &InsertOptions,
79) -> Result<()> {
80 let mut sql = format!(
81 "INSERT INTO {dest_table}{columns} SELECT * FROM {}",
82 arrow_stream_table_sql(stream_name)
83 );
84 if let Some(settings) = options.settings_clause() {
85 sql.push_str(" SETTINGS ");
86 sql.push_str(&settings);
87 }
88 conn.query(&sql, OutputFormat::Null)?;
89 Ok(())
90}
91
92fn insert_via_registered_array(
93 conn: &Connection,
94 dest_table: &str,
95 stream_name: &str,
96 arrow_schema: &ArrowSchema,
97 arrow_array: &ArrowArray,
98 columns: &str,
99 options: &InsertOptions,
100) -> Result<()> {
101 conn.register_arrow_array(stream_name, arrow_schema, arrow_array)?;
102 let result = insert_from_registered(conn, dest_table, stream_name, columns, options);
103 finish_registered_insert(result, conn.unregister_arrow_table(stream_name))
104}
105
106fn finish_registered_insert(result: Result<()>, cleanup: Result<()>) -> Result<()> {
107 match (result, cleanup) {
108 (Ok(()), Ok(())) => Ok(()),
109 (Err(err), Ok(())) | (Ok(()), Err(err)) => Err(err),
110 (Err(err), Err(cleanup_err)) => Err(Error::QueryError(format!(
111 "{err}; Arrow source cleanup also failed: {cleanup_err}"
112 ))),
113 }
114}
115
116/// Insert a single [`RecordBatch`] into `dest_table` via `register_arrow_array`.
117///
118/// `stream_name` is a temporary registration label (not a MergeTree table name).
119/// `dest_table` must be a bare ClickHouse table identifier (e.g. `metrics`);
120/// dotted names (`db.table`) and reserved identifiers are not quoted automatically.
121/// Uses the fastest one-shot Arrow path for a single in-memory batch.
122///
123/// # Examples
124///
125/// ```no_run
126/// use std::sync::Arc;
127/// use arrow::array::{Int64Array, RecordBatch};
128/// use arrow::datatypes::{DataType, Field, Schema};
129/// use chdb_rust::arrow_insert::insert_record_batch;
130/// use chdb_rust::InsertOptions;
131/// use chdb_rust::connection::Connection;
132///
133/// let conn = Connection::open_in_memory()?;
134/// conn.query(
135/// "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
136/// chdb_rust::format::OutputFormat::TabSeparated,
137/// )?;
138///
139/// let batch = RecordBatch::try_new(
140/// Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])),
141/// vec![Arc::new(Int64Array::from(vec![1i64, 2, 3]))],
142/// )
143/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?;
144///
145/// insert_record_batch(&conn, "metrics", "flush_1", batch, InsertOptions::default_bulk())?;
146/// # Ok::<(), chdb_rust::error::Error>(())
147/// ```
148///
149/// # Errors
150///
151/// Returns an error if registration, the insert query, or unregistration fails.
152pub fn insert_record_batch(
153 conn: &Connection,
154 dest_table: &str,
155 stream_name: &str,
156 batch: RecordBatch,
157 options: InsertOptions,
158) -> Result<()> {
159 let columns = column_list_clause(batch.schema().as_ref());
160 let handles = OwnedArrowArray::from_batch(batch)?;
161 insert_via_registered_array(
162 conn,
163 dest_table,
164 stream_name,
165 &handles.schema(),
166 &handles.array(),
167 &columns,
168 &options,
169 )
170}
171
172/// Insert a single [`RecordBatch`] using the direct `chdb_insert_arrow_array` FFI when
173/// available, falling back to register + SQL + unregister otherwise.
174///
175/// Unlike [`insert_record_batch`], no temporary `stream_name` is required on the direct path.
176/// This is the recommended entry point for single-batch ingest.
177///
178/// # Examples
179///
180/// ```no_run
181/// use std::sync::Arc;
182/// use arrow::array::{Int64Array, RecordBatch};
183/// use arrow::datatypes::{DataType, Field, Schema};
184/// use chdb_rust::arrow_insert::insert_record_batch_direct;
185/// use chdb_rust::InsertOptions;
186/// use chdb_rust::connection::Connection;
187///
188/// let conn = Connection::open_in_memory()?;
189/// conn.query(
190/// "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
191/// chdb_rust::format::OutputFormat::TabSeparated,
192/// )?;
193///
194/// let batch = RecordBatch::try_new(
195/// Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])),
196/// vec![Arc::new(Int64Array::from(vec![1i64, 2, 3]))],
197/// )
198/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?;
199///
200/// insert_record_batch_direct(&conn, "metrics", batch, InsertOptions::default_bulk())?;
201/// # Ok::<(), chdb_rust::error::Error>(())
202/// ```
203pub fn insert_record_batch_direct(
204 conn: &Connection,
205 dest_table: &str,
206 batch: RecordBatch,
207 options: InsertOptions,
208) -> Result<()> {
209 let columns = column_list_clause(batch.schema().as_ref());
210 let handles = OwnedArrowArray::from_batch(batch)?;
211
212 #[cfg(direct_arrow_insert)]
213 {
214 let _ = &columns;
215 conn.insert_arrow_array(dest_table, &handles.schema(), &handles.array(), &options)
216 }
217
218 #[cfg(not(direct_arrow_insert))]
219 {
220 let stream_name = format!(
221 "__rust_direct_{}",
222 FALLBACK_STREAM_SEQ.fetch_add(1, Ordering::Relaxed)
223 );
224 insert_via_registered_array(
225 conn,
226 dest_table,
227 &stream_name,
228 &handles.schema(),
229 &handles.array(),
230 &columns,
231 &options,
232 )
233 }
234}
235
236/// Insert multiple [`RecordBatch`] values via `register_arrow_stream`.
237///
238/// `batches` must share the same schema as `schema`. Prefer this when a flush
239/// spans more than one batch. See [`insert_record_batch`] for `dest_table` constraints.
240///
241/// # Examples
242///
243/// ```no_run
244/// use std::sync::Arc;
245/// use arrow::array::{Int64Array, RecordBatch};
246/// use arrow::datatypes::{DataType, Field, Schema};
247/// use chdb_rust::arrow_insert::insert_record_batches;
248/// use chdb_rust::InsertOptions;
249/// use chdb_rust::connection::Connection;
250///
251/// let conn = Connection::open_in_memory()?;
252/// conn.query(
253/// "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
254/// chdb_rust::format::OutputFormat::TabSeparated,
255/// )?;
256///
257/// let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
258/// let batches = vec![
259/// RecordBatch::try_new(
260/// Arc::clone(&schema),
261/// vec![Arc::new(Int64Array::from(vec![1i64, 2]))],
262/// )
263/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?,
264/// RecordBatch::try_new(
265/// Arc::clone(&schema),
266/// vec![Arc::new(Int64Array::from(vec![3i64, 4]))],
267/// )
268/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?,
269/// ];
270///
271/// insert_record_batches(
272/// &conn,
273/// "metrics",
274/// "flush_multi",
275/// schema,
276/// batches,
277/// InsertOptions::default_bulk(),
278/// )?;
279/// # Ok::<(), chdb_rust::error::Error>(())
280/// ```
281pub fn insert_record_batches(
282 conn: &Connection,
283 dest_table: &str,
284 stream_name: &str,
285 schema: SchemaRef,
286 batches: Vec<RecordBatch>,
287 options: InsertOptions,
288) -> Result<()> {
289 let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
290 insert_record_batch_reader(conn, dest_table, stream_name, Box::new(reader), options)
291}
292
293/// Insert from any [`RecordBatchReader`] via `register_arrow_stream`, or direct FFI when available.
294///
295/// See [`insert_record_batch`] for `dest_table` constraints.
296///
297/// # Examples
298///
299/// ```no_run
300/// use std::sync::Arc;
301/// use arrow::array::{Int64Array, RecordBatch, RecordBatchIterator};
302/// use arrow::datatypes::{DataType, Field, Schema};
303/// use chdb_rust::arrow_insert::insert_record_batch_reader;
304/// use chdb_rust::InsertOptions;
305/// use chdb_rust::connection::Connection;
306///
307/// let conn = Connection::open_in_memory()?;
308/// conn.query(
309/// "CREATE TABLE metrics (id Int64) ENGINE=MergeTree ORDER BY id",
310/// chdb_rust::format::OutputFormat::TabSeparated,
311/// )?;
312///
313/// let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
314/// let batch = RecordBatch::try_new(
315/// Arc::clone(&schema),
316/// vec![Arc::new(Int64Array::from(vec![1i64, 2]))],
317/// )
318/// .map_err(|e| chdb_rust::error::Error::QueryError(e.to_string()))?;
319/// let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
320///
321/// insert_record_batch_reader(
322/// &conn,
323/// "metrics",
324/// "flush_reader",
325/// Box::new(reader),
326/// InsertOptions::default_bulk(),
327/// )?;
328/// # Ok::<(), chdb_rust::error::Error>(())
329/// ```
330pub fn insert_record_batch_reader(
331 conn: &Connection,
332 dest_table: &str,
333 stream_name: &str,
334 reader: Box<dyn RecordBatchReader + Send>,
335 options: InsertOptions,
336) -> Result<()> {
337 let columns = column_list_clause(reader.schema().as_ref());
338 let mut ffi_stream = FFI_ArrowArrayStream::new(reader);
339 let arrow_stream = unsafe { ArrowStream::from_raw(&mut ffi_stream) };
340
341 #[cfg(direct_arrow_insert)]
342 {
343 let _ = (stream_name, &columns);
344 conn.insert_arrow_stream(dest_table, &arrow_stream, &options)
345 }
346
347 #[cfg(not(direct_arrow_insert))]
348 {
349 conn.register_arrow_stream(stream_name, &arrow_stream)?;
350 let result = insert_from_registered(conn, dest_table, stream_name, &columns, &options);
351 finish_registered_insert(result, conn.unregister_arrow_table(stream_name))
352 }
353}