chdb_rust/arrow_stream.rs
1//! Arrow streaming support for chDB.
2//!
3//! Registered streams are queried via the ClickHouse table function
4//! `ArrowStream('name')`, not as bare table identifiers.
5//!
6//! libchdb's `chdb_arrow_scan` / `chdb_arrow_array_scan` take raw Arrow C Data
7//! Interface pointers (`ArrowArrayStream*`, `ArrowSchema*`, `ArrowArray*`).
8//! The `chdb_arrow_*` typedefs in `chdb.h` are misleading wrappers; pass pointers
9//! to the actual FFI structs (e.g. from the `arrow` crate).
10//!
11//! # Overview
12//!
13//! Arrow streaming allows you to:
14//! - Register Arrow streams as virtual table functions queryable with SQL
15//! - Register Arrow schema+array pairs as one-shot streams
16//! - Unregister names when done
17//!
18//! # Examples
19//!
20//! ```no_run
21//! use chdb_rust::arrow_stream::{arrow_stream_table_sql, ArrowStream};
22//! use chdb_rust::connection::Connection;
23//! use chdb_rust::format::OutputFormat;
24//!
25//! let conn = Connection::open_in_memory()?;
26//!
27//! // Register an Arrow stream (assuming you have an Arrow C Data Interface handle)
28//! // let arrow_stream = unsafe { ArrowStream::from_raw(stream_ptr) };
29//! // conn.register_arrow_stream("my_data", &arrow_stream)?;
30//!
31//! // Query via the ArrowStream table function, not a bare table name
32//! // let sql = format!("SELECT * FROM {}", arrow_stream_table_sql("my_data"));
33//! // let result = conn.query(&sql, OutputFormat::JSONEachRow)?;
34//!
35//! // Unregister when done
36//! // conn.unregister_arrow_table("my_data")?;
37//! # Ok::<(), chdb_rust::error::Error>(())
38//! ```
39
40use crate::bindings;
41use crate::connection::Connection;
42use crate::error::Result;
43use crate::format::OutputFormat;
44use crate::query_result::QueryResult;
45use arrow::array::RecordBatchReader;
46use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
47use arrow::ffi_stream::FFI_ArrowArrayStream;
48use std::sync::atomic::{AtomicU64, Ordering};
49
50static NEXT_REGISTRATION: AtomicU64 = AtomicU64::new(0);
51
52/// A scoped, owned Arrow source registered with chDB's process-wide registry.
53///
54/// The reader and its C Stream stay alive until the registration is removed.
55/// Streaming queries borrow this handle mutably, so it cannot be closed or
56/// dropped while such a query is still being consumed. The raw registration
57/// methods remain available for callers managing C Data pointers themselves.
58/// Do not unregister its opaque name through another connection: core's
59/// registry is process-wide and cannot guard against that external action.
60#[must_use = "keep the registration alive until its queries finish"]
61pub struct ArrowRegistration<'a> {
62 conn: &'a mut Connection,
63 name: String,
64 source: Option<Box<FFI_ArrowArrayStream>>,
65 registered: bool,
66}
67
68impl<'a> ArrowRegistration<'a> {
69 pub(crate) fn new(
70 conn: &'a mut Connection,
71 reader: Box<dyn RecordBatchReader + Send>,
72 ) -> Result<Self> {
73 let mut source = Box::new(FFI_ArrowArrayStream::new(reader));
74 let name = format!(
75 "__chdb_rust_{}_{}",
76 std::process::id(),
77 NEXT_REGISTRATION.fetch_add(1, Ordering::Relaxed)
78 );
79 let raw = unsafe { ArrowStream::from_raw(source.as_mut()) };
80 conn.register_arrow_stream(&name, &raw)?;
81 Ok(Self {
82 conn,
83 name,
84 source: Some(source),
85 registered: true,
86 })
87 }
88
89 /// SQL table expression for this source, e.g. `ArrowStream('...')`.
90 pub fn sql_expr(&self) -> String {
91 arrow_stream_table_sql(&self.name)
92 }
93
94 /// Run a synchronous query while the source remains registered.
95 pub fn query(&self, sql: &str, format: OutputFormat) -> Result<QueryResult> {
96 self.conn.query(sql, format)
97 }
98
99 /// Materialize an Arrow result while the source remains registered.
100 pub fn query_arrow(&self, sql: &str) -> Result<crate::arrow_query_stream::ArrowReader> {
101 self.conn.query_arrow(sql)
102 }
103
104 /// Run a lazy Arrow query. The registration cannot be dropped until this
105 /// stream is consumed, cancelled, or dropped.
106 pub fn query_stream_arrow(
107 &mut self,
108 sql: &str,
109 ) -> Result<crate::arrow_query_stream::ArrowQueryStream<'_>> {
110 self.conn.query_stream_arrow(sql)
111 }
112
113 /// Run a lazy text query under the same lifetime constraint.
114 pub fn query_stream(
115 &mut self,
116 sql: &str,
117 format: OutputFormat,
118 ) -> Result<crate::query_stream::QueryStream<'_>> {
119 self.conn.query_stream(sql, format)
120 }
121
122 /// Unregister explicitly, returning an error if the engine refuses it.
123 /// Dropping the handle also unregisters, but cannot report that error.
124 pub fn close(mut self) -> Result<()> {
125 self.unregister()
126 }
127
128 fn unregister(&mut self) -> Result<()> {
129 if self.registered {
130 self.conn.unregister_arrow_table(&self.name)?;
131 self.registered = false;
132 }
133 Ok(())
134 }
135}
136
137impl Drop for ArrowRegistration<'_> {
138 fn drop(&mut self) {
139 if self.unregister().is_err() {
140 // Core may still hold the borrowed pointer. Leaking the source is
141 // preferable to freeing memory that a subsequent scan may read.
142 if let Some(source) = self.source.take() {
143 std::mem::forget(source);
144 }
145 }
146 }
147}
148
149/// Raw Arrow C Data Interface stream pointer expected by `chdb_arrow_scan`.
150pub type RawArrowArrayStream = *mut FFI_ArrowArrayStream;
151
152/// Raw Arrow C Data Interface schema pointer expected by `chdb_arrow_array_scan`.
153pub type RawArrowSchema = *mut FFI_ArrowSchema;
154
155/// Raw Arrow C Data Interface array pointer expected by `chdb_arrow_array_scan`.
156pub type RawArrowArray = *mut FFI_ArrowArray;
157
158/// A handle to an Arrow C Data Interface `ArrowArrayStream`.
159///
160/// Arrow streams are typically created from the `arrow` crate or other Arrow
161/// implementations that export the C Data Interface.
162///
163/// # Safety
164///
165/// The pointer must remain valid until the stream is unregistered or the
166/// connection is closed. chDB does not take ownership (`chdb_arrow_scan` passes
167/// `is_owner = false`).
168#[derive(Debug, Clone, Copy)]
169pub struct ArrowStream {
170 inner: RawArrowArrayStream,
171}
172
173unsafe impl Send for ArrowStream {}
174
175impl ArrowStream {
176 /// Create an [`ArrowStream`] from a raw `ArrowArrayStream` pointer.
177 ///
178 /// # Safety
179 ///
180 /// `stream` must be a valid, non-null `ArrowArrayStream` for the lifetime of
181 /// this handle (and until chDB unregisters the name).
182 ///
183 /// # Arguments
184 ///
185 /// * `stream` - A raw pointer to an `FFI_ArrowArrayStream`
186 ///
187 /// # Examples
188 ///
189 /// ```no_run
190 /// use chdb_rust::arrow_stream::ArrowStream;
191 ///
192 /// // Assuming you have a valid Arrow C Data Interface stream pointer
193 /// // let stream_ptr: *mut arrow::ffi_stream::FFI_ArrowArrayStream = ...;
194 /// // let arrow_stream = unsafe { ArrowStream::from_raw(stream_ptr) };
195 /// ```
196 pub unsafe fn from_raw(stream: RawArrowArrayStream) -> Self {
197 Self { inner: stream }
198 }
199
200 /// Returns the raw pointer passed through to `chdb_arrow_scan`.
201 ///
202 /// # Returns
203 ///
204 /// Returns the raw `ArrowArrayStream` pointer as [`RawArrowArrayStream`].
205 pub fn as_raw(&self) -> RawArrowArrayStream {
206 self.inner
207 }
208
209 /// The same pointer typed for the chDB C ABI.
210 pub(crate) fn as_c(&self) -> bindings::chdb_arrow_stream {
211 self.inner.cast()
212 }
213}
214
215/// A handle to an Arrow C Data Interface `ArrowSchema`.
216///
217/// Arrow schemas define the structure of Arrow data.
218///
219/// # Safety
220///
221/// The pointer must remain valid through registration and any queries that read
222/// the registered stream.
223#[derive(Debug, Clone, Copy)]
224pub struct ArrowSchema {
225 inner: RawArrowSchema,
226}
227
228unsafe impl Send for ArrowSchema {}
229
230impl ArrowSchema {
231 /// Create an [`ArrowSchema`] from a raw `ArrowSchema` pointer.
232 ///
233 /// # Safety
234 ///
235 /// `schema` must be a valid, non-null `ArrowSchema` pointer.
236 ///
237 /// # Arguments
238 ///
239 /// * `schema` - A raw pointer to an `FFI_ArrowSchema`
240 ///
241 /// # Examples
242 ///
243 /// ```no_run
244 /// use chdb_rust::arrow_stream::ArrowSchema;
245 ///
246 /// // Assuming you have a valid Arrow C Data Interface schema pointer
247 /// // let schema_ptr: *mut arrow::ffi::FFI_ArrowSchema = ...;
248 /// // let arrow_schema = unsafe { ArrowSchema::from_raw(schema_ptr) };
249 /// ```
250 pub unsafe fn from_raw(schema: RawArrowSchema) -> Self {
251 Self { inner: schema }
252 }
253
254 /// Returns the raw pointer passed through to `chdb_arrow_array_scan`.
255 ///
256 /// # Returns
257 ///
258 /// Returns the raw `ArrowSchema` pointer.
259 pub fn as_raw(&self) -> RawArrowSchema {
260 self.inner
261 }
262
263 /// The same pointer typed for the chDB C ABI.
264 pub(crate) fn as_c(&self) -> bindings::chdb_arrow_schema {
265 self.inner.cast()
266 }
267}
268
269/// A handle to an Arrow C Data Interface `ArrowArray`.
270///
271/// Arrow arrays contain the actual data in Arrow format.
272///
273/// # Safety
274///
275/// The pointer must remain valid through registration and any queries that read
276/// the registered stream.
277#[derive(Debug, Clone, Copy)]
278pub struct ArrowArray {
279 inner: RawArrowArray,
280}
281
282unsafe impl Send for ArrowArray {}
283
284impl ArrowArray {
285 /// Create an [`ArrowArray`] from a raw `ArrowArray` pointer.
286 ///
287 /// # Safety
288 ///
289 /// `array` must be a valid, non-null `ArrowArray` pointer.
290 ///
291 /// # Arguments
292 ///
293 /// * `array` - A raw pointer to an `FFI_ArrowArray`
294 ///
295 /// # Examples
296 ///
297 /// ```no_run
298 /// use chdb_rust::arrow_stream::ArrowArray;
299 ///
300 /// // Assuming you have a valid Arrow C Data Interface array pointer
301 /// // let array_ptr: *mut arrow::ffi::FFI_ArrowArray = ...;
302 /// // let arrow_array = unsafe { ArrowArray::from_raw(array_ptr) };
303 /// ```
304 pub unsafe fn from_raw(array: RawArrowArray) -> Self {
305 Self { inner: array }
306 }
307
308 /// Returns the raw pointer passed through to `chdb_arrow_array_scan`.
309 ///
310 /// # Returns
311 ///
312 /// Returns the raw `ArrowArray` pointer.
313 pub fn as_raw(&self) -> RawArrowArray {
314 self.inner
315 }
316
317 /// The same pointer typed for the chDB C ABI.
318 pub(crate) fn as_c(&self) -> bindings::chdb_arrow_array {
319 self.inner.cast()
320 }
321}
322
323/// SQL table-expression for a registered Arrow stream name.
324///
325/// Registered names are exposed as the `ArrowStream` table function, not as
326/// ordinary tables. Use this when building `INSERT ... SELECT` or ad-hoc queries.
327///
328/// # Example
329///
330/// ```
331/// use chdb_rust::arrow_stream::arrow_stream_table_sql;
332///
333/// let sql = format!(
334/// "INSERT INTO dest SELECT * FROM {}",
335/// arrow_stream_table_sql("my_batch")
336/// );
337/// assert_eq!(sql, "INSERT INTO dest SELECT * FROM ArrowStream('my_batch')");
338/// ```
339pub fn arrow_stream_table_sql(table_name: &str) -> String {
340 let escaped = table_name.replace('\'', "''");
341 format!("ArrowStream('{escaped}')")
342}
343
344#[cfg(test)]
345mod tests {
346 use super::*;
347
348 #[test]
349 fn arrow_stream_table_sql_escapes_quotes() {
350 assert_eq!(arrow_stream_table_sql("a'b"), "ArrowStream('a''b')");
351 }
352
353 #[test]
354 fn arrow_stream_from_raw_accepts_null() {
355 let stream = unsafe { ArrowStream::from_raw(std::ptr::null_mut()) };
356 assert!(stream.as_raw().is_null());
357 }
358
359 #[test]
360 fn arrow_schema_from_raw_accepts_null() {
361 let schema = unsafe { ArrowSchema::from_raw(std::ptr::null_mut()) };
362 assert!(schema.as_raw().is_null());
363 }
364
365 #[test]
366 fn arrow_array_from_raw_accepts_null() {
367 let array = unsafe { ArrowArray::from_raw(std::ptr::null_mut()) };
368 assert!(array.as_raw().is_null());
369 }
370}