Skip to main content

chdb_rust/
arrow_query_stream.rs

1//! Arrow C Data Interface batch streaming for chDB query results.
2//!
3//! This module wraps `chdb_stream_query_arrow` / `chdb_stream_fetch_arrow` and
4//! yields [`arrow::record_batch::RecordBatch`] values one engine block at a time,
5//! without Arrow IPC serialization.
6//!
7//! Available when the crate is built with the `arrow` feature.
8
9use std::ffi::CStr;
10use std::mem::ManuallyDrop;
11use std::os::raw::c_char;
12
13use arrow::array::RecordBatchReader;
14use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
15use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
16use arrow::record_batch::RecordBatch;
17
18use crate::arrow_options::ArrowOptions;
19use crate::bindings;
20use crate::connection::Connection;
21use crate::error::{Error, Result};
22use crate::query_param::{EncodedParams, QueryParams};
23use crate::query_result::QueryResult;
24
25enum ArrowQueryStreamConnection<'a> {
26    Borrowed(&'a mut Connection),
27    Owned(Connection),
28}
29
30/// A one-shot Arrow reader over a query's entire result.
31///
32/// Returned by [`Connection::query_arrow`](crate::connection::Connection::query_arrow)
33/// and [`Connection::query_arrow_with_opts`](crate::connection::Connection::query_arrow_with_opts).
34///
35/// The C ABI transfers ownership of the stream's `release` callback to the
36/// caller. The engine materializes a standalone Arrow table, so this reader
37/// does not borrow the [`Connection`] that produced it and can be drained
38/// after that connection is dropped.
39pub struct ArrowReader {
40    inner: ArrowArrayStreamReader,
41}
42
43impl ArrowReader {
44    pub(crate) fn new(inner: ArrowArrayStreamReader) -> Self {
45        Self { inner }
46    }
47}
48
49impl Iterator for ArrowReader {
50    type Item = arrow::error::Result<RecordBatch>;
51
52    fn next(&mut self) -> Option<Self::Item> {
53        self.inner.next()
54    }
55}
56
57impl arrow::array::RecordBatchReader for ArrowReader {
58    fn schema(&self) -> arrow::datatypes::SchemaRef {
59        self.inner.schema()
60    }
61}
62
63/// A streaming Arrow query that yields [`RecordBatch`] values one block at a time.
64///
65/// Batches come from the Arrow C Data Interface (no Arrow IPC). Create a stream from a
66/// [`Connection`], [`Session`](crate::session::Session), or the crate-level
67/// `execute_stream_arrow*` helpers. Implements [`Iterator`].
68///
69/// # Thread Safety
70///
71/// While a borrowed stream is active, its [`Connection`]
72/// is exclusively borrowed and cannot be used for other queries. This prevents concurrent
73/// access to a non-[`Sync`] handle. Streams from [`execute_stream_arrow`](crate::execute_stream_arrow)
74/// own their connection outright.
75pub struct ArrowQueryStream<'a> {
76    conn: ArrowQueryStreamConnection<'a>,
77    inner: *mut bindings::chdb_result,
78    finished: bool,
79    schema: Option<arrow::datatypes::SchemaRef>,
80}
81
82impl<'a> ArrowQueryStream<'a> {
83    pub(crate) fn start_borrowed(
84        conn: &'a mut Connection,
85        sql: &str,
86        opts: Option<&ArrowOptions>,
87    ) -> Result<Self> {
88        let inner = Self::start_query(conn.handle(), sql, opts)?;
89        Ok(Self {
90            conn: ArrowQueryStreamConnection::Borrowed(conn),
91            inner,
92            finished: false,
93            schema: None,
94        })
95    }
96
97    pub(crate) fn start_owned(
98        conn: Connection,
99        sql: &str,
100        opts: Option<&ArrowOptions>,
101    ) -> Result<Self> {
102        let inner = Self::start_query(conn.handle(), sql, opts)?;
103        Ok(Self {
104            conn: ArrowQueryStreamConnection::Owned(conn),
105            inner,
106            finished: false,
107            schema: None,
108        })
109    }
110
111    pub(crate) fn start_borrowed_with_params(
112        conn: &'a mut Connection,
113        sql: &str,
114        params: impl Into<QueryParams>,
115        opts: Option<&ArrowOptions>,
116    ) -> Result<Self> {
117        let inner = Self::start_query_with_params(conn.handle(), sql, params, opts)?;
118        Ok(Self {
119            conn: ArrowQueryStreamConnection::Borrowed(conn),
120            inner,
121            finished: false,
122            schema: None,
123        })
124    }
125
126    pub(crate) fn start_owned_with_params(
127        conn: Connection,
128        sql: &str,
129        params: impl Into<QueryParams>,
130        opts: Option<&ArrowOptions>,
131    ) -> Result<Self> {
132        let inner = Self::start_query_with_params(conn.handle(), sql, params, opts)?;
133        Ok(Self {
134            conn: ArrowQueryStreamConnection::Owned(conn),
135            inner,
136            finished: false,
137            schema: None,
138        })
139    }
140
141    fn start_query(
142        conn: bindings::chdb_connection,
143        sql: &str,
144        opts: Option<&ArrowOptions>,
145    ) -> Result<*mut bindings::chdb_result> {
146        // chdb_stream_query_arrow_n takes pointer + length for the query text,
147        // so it does not have to be NUL-terminated. The C struct must outlive
148        // the call, so it is materialized here rather than in a temporary
149        // inside the argument list; a null options pointer asks for the
150        // engine's default type mapping. It returns an owned streaming
151        // chdb_result handle (or null on failure) that the caller must both
152        // cancel with chdb_stream_cancel_query and free with
153        // chdb_destroy_query_result once done — ArrowQueryStream::cancel
154        // (called from Drop) does both. `check_start`, below, probes the
155        // handle for a start-up error without taking ownership of it.
156        let c_opts = opts.map(|o| o.to_c());
157        let opts_ptr = c_opts.as_ref().map_or(std::ptr::null(), |o| {
158            o as *const bindings::chdb_arrow_options
159        });
160
161        let stream_ptr = unsafe {
162            bindings::chdb_stream_query_arrow_n(
163                conn,
164                sql.as_ptr() as *const c_char,
165                sql.len(),
166                opts_ptr,
167            )
168        };
169
170        Self::check_start(stream_ptr)
171    }
172
173    /// Wraps `chdb_stream_query_arrow_with_params_n`. A null options pointer
174    /// asks for the engine's default type mapping. It returns an owned
175    /// streaming `chdb_result` handle (or null on failure) that the caller
176    /// must both cancel with `chdb_stream_cancel_query` and free with
177    /// `chdb_destroy_query_result` once done — `ArrowQueryStream::cancel`
178    /// (called from `Drop`) does both.
179    /// A non-null stream handle may still carry an initialisation error, so the
180    /// handle is probed once before it is handed out. The probe must not free
181    /// the handle on the success path, hence the `ManuallyDrop` dance.
182    fn check_start(stream_ptr: *mut bindings::chdb_result) -> Result<*mut bindings::chdb_result> {
183        if stream_ptr.is_null() {
184            return Err(Error::NoResult);
185        }
186
187        let probe = ManuallyDrop::new(QueryResult::new(stream_ptr));
188        if let Err(e) = probe.check_error_ref() {
189            drop(ManuallyDrop::into_inner(probe));
190            return Err(e);
191        }
192        std::mem::forget(ManuallyDrop::into_inner(probe));
193
194        Ok(stream_ptr)
195    }
196
197    fn start_query_with_params(
198        conn: bindings::chdb_connection,
199        sql: &str,
200        params: impl Into<QueryParams>,
201        opts: Option<&ArrowOptions>,
202    ) -> Result<*mut bindings::chdb_result> {
203        let encoded = EncodedParams::encode(params)?;
204        // Materialized into a named local so the pointer handed to C outlives the call.
205        let c_opts = opts.map(|o| o.to_c());
206        let opts_ptr = c_opts.as_ref().map_or(std::ptr::null(), |o| {
207            o as *const bindings::chdb_arrow_options
208        });
209
210        // chdb_stream_query_arrow_with_params_n takes pointer + length for the query
211        // and each bound value. A null options pointer selects the engine default
212        // type mapping. It returns an owned streaming chdb_result handle (or null
213        // on failure) that ArrowQueryStream::cancel (called from Drop) cancels and
214        // frees.
215        let stream_ptr = unsafe {
216            bindings::chdb_stream_query_arrow_with_params_n(
217                conn,
218                sql.as_ptr() as *const c_char,
219                sql.len(),
220                opts_ptr,
221                encoded.names_ptr(),
222                encoded.name_lens_ptr(),
223                encoded.values_ptr(),
224                encoded.value_lens_ptr(),
225                encoded.len(),
226            )
227        };
228
229        Self::check_start(stream_ptr)
230    }
231
232    fn conn_handle(&self) -> bindings::chdb_connection {
233        match &self.conn {
234            ArrowQueryStreamConnection::Borrowed(conn) => conn.handle(),
235            ArrowQueryStreamConnection::Owned(conn) => conn.handle(),
236        }
237    }
238
239    /// Fetch the next Arrow record batch from the query stream.
240    ///
241    /// The schema, once fetched, remains available after this returns `None`.
242    /// Older engines may not export a schema for a zero-row result.
243    pub fn schema(&self) -> Option<arrow::datatypes::SchemaRef> {
244        self.schema.clone()
245    }
246
247    /// Returns `Ok(None)` when the stream is exhausted, including empty result sets.
248    pub fn next_batch(&mut self) -> Result<Option<RecordBatch>> {
249        if self.finished || self.inner.is_null() {
250            return Ok(None);
251        }
252
253        let mut ffi_stream = FFI_ArrowArrayStream::empty();
254        let out_ptr = (&mut ffi_stream as *mut FFI_ArrowArrayStream).cast();
255
256        let state =
257            unsafe { bindings::chdb_stream_fetch_arrow(self.conn_handle(), self.inner, out_ptr) };
258
259        if state != bindings::chdb_state_CHDBSuccess {
260            let err_msg = unsafe { bindings::chdb_result_error(self.inner) };
261            let detail = if err_msg.is_null() {
262                "chdb_stream_fetch_arrow failed".to_string()
263            } else {
264                unsafe { CStr::from_ptr(err_msg) }
265                    .to_string_lossy()
266                    .into_owned()
267            };
268            self.cancel();
269            return Err(Error::QueryError(detail));
270        }
271
272        match is_eos_without_schema(&mut ffi_stream) {
273            Ok(true) => {
274                self.finished = true;
275                return Ok(None);
276            }
277            Ok(false) => {}
278            Err(err) => {
279                self.cancel();
280                return Err(err);
281            }
282        }
283
284        let mut reader = match ArrowArrayStreamReader::try_new(ffi_stream) {
285            Ok(reader) => reader,
286            Err(e) => {
287                self.cancel();
288                return Err(Error::QueryError(e.to_string()));
289            }
290        };
291
292        self.schema = Some(reader.schema());
293
294        match reader.next() {
295            Some(Ok(batch)) => Ok(Some(batch)),
296            Some(Err(e)) => {
297                self.cancel();
298                Err(Error::QueryError(e.to_string()))
299            }
300            None => {
301                self.finished = true;
302                Ok(None)
303            }
304        }
305    }
306
307    /// Cancel the streaming query and release resources.
308    pub fn cancel(&mut self) {
309        if self.inner.is_null() {
310            return;
311        }
312
313        if !self.finished {
314            unsafe {
315                bindings::chdb_stream_cancel_query(self.conn_handle(), self.inner);
316            }
317        }
318        unsafe {
319            bindings::chdb_destroy_query_result(self.inner);
320        }
321        self.inner = std::ptr::null_mut();
322        self.finished = true;
323    }
324}
325
326/// On older engines EOS may be a stream without a schema. A failed schema
327/// callback alone is not proof of EOS: check get_next before deciding whether
328/// to suppress the error. Both probe objects release themselves on drop.
329fn is_eos_without_schema(ffi_stream: &mut FFI_ArrowArrayStream) -> Result<bool> {
330    let Some(get_schema) = ffi_stream.get_schema else {
331        return Err(Error::InvalidData(
332            "Arrow stream has no get_schema callback".into(),
333        ));
334    };
335    let mut schema = FFI_ArrowSchema::empty();
336    let ret_code = unsafe { get_schema(ffi_stream, &mut schema) };
337    if ret_code == 0 {
338        return Ok(false);
339    }
340
341    let Some(get_next) = ffi_stream.get_next else {
342        return Err(Error::InvalidData(
343            "Arrow stream has no get_next callback".into(),
344        ));
345    };
346    let mut array = FFI_ArrowArray::empty();
347    let next_code = unsafe { get_next(ffi_stream, &mut array) };
348    let detail = ffi_stream.get_last_error.and_then(|get_error| {
349        let ptr = unsafe { get_error(ffi_stream) };
350        (!ptr.is_null()).then(|| {
351            unsafe { CStr::from_ptr(ptr) }
352                .to_string_lossy()
353                .into_owned()
354        })
355    });
356    if next_code == 0 && array.release.is_none() && detail.is_none() {
357        return Ok(true);
358    }
359    Err(Error::QueryError(detail.unwrap_or_else(|| {
360        format!("Arrow stream schema failed ({ret_code}); next batch returned {next_code}")
361    })))
362}
363
364impl Iterator for ArrowQueryStream<'_> {
365    type Item = Result<RecordBatch>;
366
367    fn next(&mut self) -> Option<Self::Item> {
368        match self.next_batch() {
369            Ok(Some(batch)) => Some(Ok(batch)),
370            Ok(None) => None,
371            Err(err) => Some(Err(err)),
372        }
373    }
374}
375
376impl Drop for ArrowQueryStream<'_> {
377    fn drop(&mut self) {
378        if !self.inner.is_null() {
379            self.cancel();
380        }
381    }
382}
383
384#[cfg(test)]
385mod tests {
386    use super::*;
387    use crate::session::SessionBuilder;
388    use crate::test_utils::tempdir;
389
390    unsafe extern "C" fn failing_schema(
391        _: *mut FFI_ArrowArrayStream,
392        _: *mut FFI_ArrowSchema,
393    ) -> i32 {
394        1
395    }
396
397    unsafe extern "C" fn empty_next(_: *mut FFI_ArrowArrayStream, _: *mut FFI_ArrowArray) -> i32 {
398        0
399    }
400
401    unsafe extern "C" fn schema_error(_: *mut FFI_ArrowArrayStream) -> *const c_char {
402        c"invalid schema".as_ptr()
403    }
404
405    #[test]
406    fn schema_error_is_not_silently_treated_as_eof() {
407        let mut stream = FFI_ArrowArrayStream::empty();
408        stream.get_schema = Some(failing_schema);
409        stream.get_next = Some(empty_next);
410        stream.get_last_error = Some(schema_error);
411        let err = is_eos_without_schema(&mut stream).unwrap_err();
412        assert!(err.to_string().contains("invalid schema"));
413
414        stream.get_last_error = None;
415        assert!(is_eos_without_schema(&mut stream).unwrap());
416    }
417
418    #[test]
419    fn test_arrow_query_stream_row_count_and_chunking() -> Result<()> {
420        let mut conn = Connection::open_in_memory()?;
421        let mut stream = conn.query_stream_arrow("SELECT number FROM numbers(100_000)")?;
422
423        let mut batches = 0usize;
424        let mut rows = 0usize;
425        while let Some(batch) = stream.next_batch()? {
426            batches += 1;
427            rows += batch.num_rows();
428        }
429
430        assert_eq!(rows, 100_000);
431        assert!(batches > 1, "expected multiple batches, got {batches}");
432        Ok(())
433    }
434
435    #[test]
436    fn test_execute_stream_arrow() -> Result<()> {
437        use crate::execute_stream_arrow;
438
439        let mut stream = execute_stream_arrow("SELECT number FROM numbers(10)")?;
440        let mut rows = 0usize;
441        while let Some(batch) = stream.next_batch()? {
442            rows += batch.num_rows();
443        }
444
445        assert_eq!(rows, 10);
446        Ok(())
447    }
448
449    #[test]
450    fn test_session_execute_stream_arrow() -> Result<()> {
451        let tmp = tempdir();
452        let mut session = SessionBuilder::new()
453            .with_data_path(tmp.path())
454            .with_auto_cleanup(true)
455            .build()?;
456
457        session.execute(
458            "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
459            None,
460        )?;
461        session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
462
463        let mut stream = session.execute_stream_arrow("SELECT * FROM items")?;
464        let mut rows = 0usize;
465        while let Some(batch) = stream.next_batch()? {
466            rows += batch.num_rows();
467        }
468
469        assert_eq!(rows, 3);
470        Ok(())
471    }
472
473    #[test]
474    fn test_arrow_query_stream_iterator() -> Result<()> {
475        let mut conn = Connection::open_in_memory()?;
476        let stream = conn.query_stream_arrow("SELECT number FROM numbers(5)")?;
477
478        let batches: Vec<_> = stream.collect::<Result<Vec<_>>>()?;
479        assert_eq!(batches.len(), 1);
480        assert_eq!(batches[0].num_rows(), 5);
481        Ok(())
482    }
483
484    #[test]
485    fn test_arrow_query_stream_empty_filter_returns_none() -> Result<()> {
486        let tmp = tempdir();
487        let mut session = SessionBuilder::new()
488            .with_data_path(tmp.path())
489            .with_auto_cleanup(true)
490            .build()?;
491
492        session.execute(
493            "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
494            None,
495        )?;
496        session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
497
498        let mut stream = session.execute_stream_arrow("SELECT * FROM items WHERE id > 100")?;
499        assert!(stream.next_batch()?.is_none());
500        assert!(stream.next_batch()?.is_none());
501        Ok(())
502    }
503
504    #[test]
505    fn test_arrow_query_stream_error_then_retry_returns_none() -> Result<()> {
506        let mut conn = Connection::open_in_memory()?;
507        let mut stream = conn.query_stream_arrow("SELECT * FROM nonexistent_table")?;
508
509        assert!(stream.next_batch().is_err());
510        assert!(stream.next_batch()?.is_none());
511        Ok(())
512    }
513
514    #[test]
515    fn test_arrow_query_stream_syntax_error_fails_at_start() -> Result<()> {
516        let mut conn = Connection::open_in_memory()?;
517        let result = conn.query_stream_arrow("SELECT invalid syntax here");
518        assert!(result.is_err());
519        Ok(())
520    }
521
522    #[test]
523    fn test_arrow_query_stream_with_params_empty_filter_returns_none() -> Result<()> {
524        let tmp = tempdir();
525        let mut session = SessionBuilder::new()
526            .with_data_path(tmp.path())
527            .with_auto_cleanup(true)
528            .build()?;
529
530        session.execute(
531            "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
532            None,
533        )?;
534        session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
535
536        let mut stream = session.execute_stream_arrow_with_params(
537            "SELECT * FROM items WHERE id > {min_id:UInt64}",
538            [("min_id", 100_u64)],
539            None,
540        )?;
541        assert!(stream.next_batch()?.is_none());
542        assert!(stream.next_batch()?.is_none());
543        Ok(())
544    }
545
546    #[test]
547    fn test_arrow_query_stream_with_params_error_then_retry_returns_none() -> Result<()> {
548        let mut conn = Connection::open_in_memory()?;
549        let mut stream = conn.query_stream_arrow_with_params(
550            "SELECT * FROM nonexistent_table WHERE id = {id:UInt64}",
551            [("id", 1_u64)],
552            None,
553        )?;
554
555        assert!(stream.next_batch().is_err());
556        assert!(stream.next_batch()?.is_none());
557        Ok(())
558    }
559
560    #[test]
561    fn test_arrow_query_stream_with_params_syntax_error_fails_at_start() -> Result<()> {
562        let mut conn = Connection::open_in_memory()?;
563        let result =
564            conn.query_stream_arrow_with_params("SELECT invalid syntax here", [("x", 1_u64)], None);
565        assert!(result.is_err());
566        Ok(())
567    }
568
569    #[test]
570    fn test_arrow_query_stream_early_drop() -> Result<()> {
571        let mut conn = Connection::open_in_memory()?;
572        let mut stream = conn.query_stream_arrow("SELECT number FROM numbers(1_000_000)")?;
573        let first = stream.next_batch()?.expect("expected a batch");
574        assert!(first.num_rows() > 0);
575        drop(stream);
576        Ok(())
577    }
578}