Skip to main content

chdb_rust/
arrow_stream.rs

1//! Arrow streaming support for chDB.
2//!
3//! Registered streams are queried via the ClickHouse table function
4//! `ArrowStream('name')`, not as bare table identifiers.
5//!
6//! libchdb's `chdb_arrow_scan` / `chdb_arrow_array_scan` take raw Arrow C Data
7//! Interface pointers (`ArrowArrayStream*`, `ArrowSchema*`, `ArrowArray*`).
8//! The `chdb_arrow_*` typedefs in `chdb.h` are misleading wrappers; pass pointers
9//! to the actual FFI structs (e.g. from the `arrow` crate).
10//!
11//! # Overview
12//!
13//! Arrow streaming allows you to:
14//! - Register Arrow streams as virtual table functions queryable with SQL
15//! - Register Arrow schema+array pairs as one-shot streams
16//! - Unregister names when done
17//!
18//! # Examples
19//!
20//! ```no_run
21//! use chdb_rust::arrow_stream::{arrow_stream_table_sql, ArrowStream};
22//! use chdb_rust::connection::Connection;
23//! use chdb_rust::format::OutputFormat;
24//!
25//! let conn = Connection::open_in_memory()?;
26//!
27//! // Register an Arrow stream (assuming you have an Arrow C Data Interface handle)
28//! // let arrow_stream = unsafe { ArrowStream::from_raw(stream_ptr) };
29//! // conn.register_arrow_stream("my_data", &arrow_stream)?;
30//!
31//! // Query via the ArrowStream table function, not a bare table name
32//! // let sql = format!("SELECT * FROM {}", arrow_stream_table_sql("my_data"));
33//! // let result = conn.query(&sql, OutputFormat::JSONEachRow)?;
34//!
35//! // Unregister when done
36//! // conn.unregister_arrow_table("my_data")?;
37//! # Ok::<(), chdb_rust::error::Error>(())
38//! ```
39
40use crate::bindings;
41use crate::connection::Connection;
42use crate::error::Result;
43use crate::format::OutputFormat;
44use crate::query_result::QueryResult;
45use arrow::array::RecordBatchReader;
46use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
47use arrow::ffi_stream::FFI_ArrowArrayStream;
48use std::sync::atomic::{AtomicU64, Ordering};
49
50static NEXT_REGISTRATION: AtomicU64 = AtomicU64::new(0);
51
52/// A scoped, owned Arrow source registered with chDB's process-wide registry.
53///
54/// The reader and its C Stream stay alive until the registration is removed.
55/// Streaming queries borrow this handle mutably, so it cannot be closed or
56/// dropped while such a query is still being consumed. The raw registration
57/// methods remain available for callers managing C Data pointers themselves.
58/// Do not unregister its opaque name through another connection: core's
59/// registry is process-wide and cannot guard against that external action.
60#[must_use = "keep the registration alive until its queries finish"]
61pub struct ArrowRegistration<'a> {
62    conn: &'a mut Connection,
63    name: String,
64    source: Option<Box<FFI_ArrowArrayStream>>,
65    registered: bool,
66}
67
68impl<'a> ArrowRegistration<'a> {
69    pub(crate) fn new(
70        conn: &'a mut Connection,
71        reader: Box<dyn RecordBatchReader + Send>,
72    ) -> Result<Self> {
73        let mut source = Box::new(FFI_ArrowArrayStream::new(reader));
74        let name = format!(
75            "__chdb_rust_{}_{}",
76            std::process::id(),
77            NEXT_REGISTRATION.fetch_add(1, Ordering::Relaxed)
78        );
79        let raw = unsafe { ArrowStream::from_raw(source.as_mut()) };
80        conn.register_arrow_stream(&name, &raw)?;
81        Ok(Self {
82            conn,
83            name,
84            source: Some(source),
85            registered: true,
86        })
87    }
88
89    /// SQL table expression for this source, e.g. `ArrowStream('...')`.
90    pub fn sql_expr(&self) -> String {
91        arrow_stream_table_sql(&self.name)
92    }
93
94    /// Run a synchronous query while the source remains registered.
95    pub fn query(&self, sql: &str, format: OutputFormat) -> Result<QueryResult> {
96        self.conn.query(sql, format)
97    }
98
99    /// Materialize an Arrow result while the source remains registered.
100    pub fn query_arrow(&self, sql: &str) -> Result<crate::arrow_query_stream::ArrowReader> {
101        self.conn.query_arrow(sql)
102    }
103
104    /// Run a lazy Arrow query. The registration cannot be dropped until this
105    /// stream is consumed, cancelled, or dropped.
106    pub fn query_stream_arrow(
107        &mut self,
108        sql: &str,
109    ) -> Result<crate::arrow_query_stream::ArrowQueryStream<'_>> {
110        self.conn.query_stream_arrow(sql)
111    }
112
113    /// Run a lazy text query under the same lifetime constraint.
114    pub fn query_stream(
115        &mut self,
116        sql: &str,
117        format: OutputFormat,
118    ) -> Result<crate::query_stream::QueryStream<'_>> {
119        self.conn.query_stream(sql, format)
120    }
121
122    /// Unregister explicitly, returning an error if the engine refuses it.
123    /// Dropping the handle also unregisters, but cannot report that error.
124    pub fn close(mut self) -> Result<()> {
125        self.unregister()
126    }
127
128    fn unregister(&mut self) -> Result<()> {
129        if self.registered {
130            self.conn.unregister_arrow_table(&self.name)?;
131            self.registered = false;
132        }
133        Ok(())
134    }
135}
136
137impl Drop for ArrowRegistration<'_> {
138    fn drop(&mut self) {
139        if self.unregister().is_err() {
140            // Core may still hold the borrowed pointer. Leaking the source is
141            // preferable to freeing memory that a subsequent scan may read.
142            if let Some(source) = self.source.take() {
143                std::mem::forget(source);
144            }
145        }
146    }
147}
148
149/// Raw Arrow C Data Interface stream pointer expected by `chdb_arrow_scan`.
150pub type RawArrowArrayStream = *mut FFI_ArrowArrayStream;
151
152/// Raw Arrow C Data Interface schema pointer expected by `chdb_arrow_array_scan`.
153pub type RawArrowSchema = *mut FFI_ArrowSchema;
154
155/// Raw Arrow C Data Interface array pointer expected by `chdb_arrow_array_scan`.
156pub type RawArrowArray = *mut FFI_ArrowArray;
157
158/// A handle to an Arrow C Data Interface `ArrowArrayStream`.
159///
160/// Arrow streams are typically created from the `arrow` crate or other Arrow
161/// implementations that export the C Data Interface.
162///
163/// # Safety
164///
165/// The pointer must remain valid until the stream is unregistered or the
166/// connection is closed. chDB does not take ownership (`chdb_arrow_scan` passes
167/// `is_owner = false`).
168#[derive(Debug, Clone, Copy)]
169pub struct ArrowStream {
170    inner: RawArrowArrayStream,
171}
172
173unsafe impl Send for ArrowStream {}
174
175impl ArrowStream {
176    /// Create an [`ArrowStream`] from a raw `ArrowArrayStream` pointer.
177    ///
178    /// # Safety
179    ///
180    /// `stream` must be a valid, non-null `ArrowArrayStream` for the lifetime of
181    /// this handle (and until chDB unregisters the name).
182    ///
183    /// # Arguments
184    ///
185    /// * `stream` - A raw pointer to an `FFI_ArrowArrayStream`
186    ///
187    /// # Examples
188    ///
189    /// ```no_run
190    /// use chdb_rust::arrow_stream::ArrowStream;
191    ///
192    /// // Assuming you have a valid Arrow C Data Interface stream pointer
193    /// // let stream_ptr: *mut arrow::ffi_stream::FFI_ArrowArrayStream = ...;
194    /// // let arrow_stream = unsafe { ArrowStream::from_raw(stream_ptr) };
195    /// ```
196    pub unsafe fn from_raw(stream: RawArrowArrayStream) -> Self {
197        Self { inner: stream }
198    }
199
200    /// Returns the raw pointer passed through to `chdb_arrow_scan`.
201    ///
202    /// # Returns
203    ///
204    /// Returns the raw `ArrowArrayStream` pointer as [`RawArrowArrayStream`].
205    pub fn as_raw(&self) -> RawArrowArrayStream {
206        self.inner
207    }
208
209    /// The same pointer typed for the chDB C ABI.
210    pub(crate) fn as_c(&self) -> bindings::chdb_arrow_stream {
211        self.inner.cast()
212    }
213}
214
215/// A handle to an Arrow C Data Interface `ArrowSchema`.
216///
217/// Arrow schemas define the structure of Arrow data.
218///
219/// # Safety
220///
221/// The pointer must remain valid through registration and any queries that read
222/// the registered stream.
223#[derive(Debug, Clone, Copy)]
224pub struct ArrowSchema {
225    inner: RawArrowSchema,
226}
227
228unsafe impl Send for ArrowSchema {}
229
230impl ArrowSchema {
231    /// Create an [`ArrowSchema`] from a raw `ArrowSchema` pointer.
232    ///
233    /// # Safety
234    ///
235    /// `schema` must be a valid, non-null `ArrowSchema` pointer.
236    ///
237    /// # Arguments
238    ///
239    /// * `schema` - A raw pointer to an `FFI_ArrowSchema`
240    ///
241    /// # Examples
242    ///
243    /// ```no_run
244    /// use chdb_rust::arrow_stream::ArrowSchema;
245    ///
246    /// // Assuming you have a valid Arrow C Data Interface schema pointer
247    /// // let schema_ptr: *mut arrow::ffi::FFI_ArrowSchema = ...;
248    /// // let arrow_schema = unsafe { ArrowSchema::from_raw(schema_ptr) };
249    /// ```
250    pub unsafe fn from_raw(schema: RawArrowSchema) -> Self {
251        Self { inner: schema }
252    }
253
254    /// Returns the raw pointer passed through to `chdb_arrow_array_scan`.
255    ///
256    /// # Returns
257    ///
258    /// Returns the raw `ArrowSchema` pointer.
259    pub fn as_raw(&self) -> RawArrowSchema {
260        self.inner
261    }
262
263    /// The same pointer typed for the chDB C ABI.
264    pub(crate) fn as_c(&self) -> bindings::chdb_arrow_schema {
265        self.inner.cast()
266    }
267}
268
269/// A handle to an Arrow C Data Interface `ArrowArray`.
270///
271/// Arrow arrays contain the actual data in Arrow format.
272///
273/// # Safety
274///
275/// The pointer must remain valid through registration and any queries that read
276/// the registered stream.
277#[derive(Debug, Clone, Copy)]
278pub struct ArrowArray {
279    inner: RawArrowArray,
280}
281
282unsafe impl Send for ArrowArray {}
283
284impl ArrowArray {
285    /// Create an [`ArrowArray`] from a raw `ArrowArray` pointer.
286    ///
287    /// # Safety
288    ///
289    /// `array` must be a valid, non-null `ArrowArray` pointer.
290    ///
291    /// # Arguments
292    ///
293    /// * `array` - A raw pointer to an `FFI_ArrowArray`
294    ///
295    /// # Examples
296    ///
297    /// ```no_run
298    /// use chdb_rust::arrow_stream::ArrowArray;
299    ///
300    /// // Assuming you have a valid Arrow C Data Interface array pointer
301    /// // let array_ptr: *mut arrow::ffi::FFI_ArrowArray = ...;
302    /// // let arrow_array = unsafe { ArrowArray::from_raw(array_ptr) };
303    /// ```
304    pub unsafe fn from_raw(array: RawArrowArray) -> Self {
305        Self { inner: array }
306    }
307
308    /// Returns the raw pointer passed through to `chdb_arrow_array_scan`.
309    ///
310    /// # Returns
311    ///
312    /// Returns the raw `ArrowArray` pointer.
313    pub fn as_raw(&self) -> RawArrowArray {
314        self.inner
315    }
316
317    /// The same pointer typed for the chDB C ABI.
318    pub(crate) fn as_c(&self) -> bindings::chdb_arrow_array {
319        self.inner.cast()
320    }
321}
322
323/// SQL table-expression for a registered Arrow stream name.
324///
325/// Registered names are exposed as the `ArrowStream` table function, not as
326/// ordinary tables. Use this when building `INSERT ... SELECT` or ad-hoc queries.
327///
328/// # Example
329///
330/// ```
331/// use chdb_rust::arrow_stream::arrow_stream_table_sql;
332///
333/// let sql = format!(
334///     "INSERT INTO dest SELECT * FROM {}",
335///     arrow_stream_table_sql("my_batch")
336/// );
337/// assert_eq!(sql, "INSERT INTO dest SELECT * FROM ArrowStream('my_batch')");
338/// ```
339pub fn arrow_stream_table_sql(table_name: &str) -> String {
340    let escaped = table_name.replace('\'', "''");
341    format!("ArrowStream('{escaped}')")
342}
343
344#[cfg(test)]
345mod tests {
346    use super::*;
347
348    #[test]
349    fn arrow_stream_table_sql_escapes_quotes() {
350        assert_eq!(arrow_stream_table_sql("a'b"), "ArrowStream('a''b')");
351    }
352
353    #[test]
354    fn arrow_stream_from_raw_accepts_null() {
355        let stream = unsafe { ArrowStream::from_raw(std::ptr::null_mut()) };
356        assert!(stream.as_raw().is_null());
357    }
358
359    #[test]
360    fn arrow_schema_from_raw_accepts_null() {
361        let schema = unsafe { ArrowSchema::from_raw(std::ptr::null_mut()) };
362        assert!(schema.as_raw().is_null());
363    }
364
365    #[test]
366    fn arrow_array_from_raw_accepts_null() {
367        let array = unsafe { ArrowArray::from_raw(std::ptr::null_mut()) };
368        assert!(array.as_raw().is_null());
369    }
370}