Skip to main content

chdb_rust/
query_stream.rs

1//! Streaming query result handling for chDB.
2//!
3//! This module provides the [`QueryStream`] type for reading large query results
4//! in chunks without materializing the entire output in memory.
5
6use std::mem::ManuallyDrop;
7use std::os::raw::c_char;
8
9use crate::bindings;
10use crate::connection::Connection;
11use crate::error::{Error, Result};
12use crate::format::OutputFormat;
13use crate::query_param::{EncodedParams, QueryParams};
14use crate::query_result::QueryResult;
15
16enum QueryStreamConnection<'a> {
17    Borrowed(&'a mut Connection),
18    Owned(Connection),
19}
20
21/// A streaming query result that yields data in chunks.
22///
23/// `QueryStream` is returned by [`Connection::query_stream`](crate::connection::Connection::query_stream),
24/// [`Connection::query_stream_with_params`](crate::connection::Connection::query_stream_with_params),
25/// [`Session::execute_stream`](crate::session::Session::execute_stream),
26/// [`Session::execute_stream_with_params`](crate::session::Session::execute_stream_with_params),
27/// [`execute_stream`](crate::execute_stream), and
28/// [`execute_stream_with_params`](crate::execute_stream_with_params). Each chunk is a [`QueryResult`] that the caller owns
29/// and must drop before requesting the next chunk.
30///
31/// `QueryStream` also implements [`Iterator`], so you can collect chunks with adapter methods
32/// such as [`Iterator::collect`].
33///
34/// # Thread Safety
35///
36/// While a borrowed stream is active, its [`Connection`]
37/// is exclusively borrowed and cannot be used for other queries. This prevents concurrent
38/// access to a non-[`Sync`] handle. Streams from [`execute_stream`](crate::execute_stream)
39/// own their connection outright.
40///
41/// # Examples
42///
43/// ```no_run
44/// use chdb_rust::connection::Connection;
45/// use chdb_rust::format::OutputFormat;
46///
47/// let mut conn = Connection::open_in_memory()?;
48/// let mut stream = conn.query_stream(
49///     "SELECT number FROM numbers(100_000)",
50///     OutputFormat::JSONEachRow,
51/// )?;
52///
53/// while let Some(chunk) = stream.next_chunk()? {
54///     print!("{}", chunk.data_utf8_lossy());
55/// }
56/// # Ok::<(), chdb_rust::error::Error>(())
57/// ```
58pub struct QueryStream<'a> {
59    conn: QueryStreamConnection<'a>,
60    stream: *mut bindings::chdb_result,
61    finished: bool,
62}
63
64impl<'a> QueryStream<'a> {
65    pub(crate) fn start_borrowed(
66        conn: &'a mut Connection,
67        sql: &str,
68        format: OutputFormat,
69    ) -> Result<Self> {
70        let stream = Self::start_query(conn.handle(), sql, format)?;
71        Ok(Self {
72            conn: QueryStreamConnection::Borrowed(conn),
73            stream,
74            finished: false,
75        })
76    }
77
78    pub(crate) fn start_owned(conn: Connection, sql: &str, format: OutputFormat) -> Result<Self> {
79        let stream = Self::start_query(conn.handle(), sql, format)?;
80        Ok(Self {
81            conn: QueryStreamConnection::Owned(conn),
82            stream,
83            finished: false,
84        })
85    }
86
87    pub(crate) fn start_borrowed_with_params(
88        conn: &'a mut Connection,
89        sql: &str,
90        format: OutputFormat,
91        params: impl Into<QueryParams>,
92    ) -> Result<Self> {
93        let stream = Self::start_query_with_params(conn.handle(), sql, format, params)?;
94        Ok(Self {
95            conn: QueryStreamConnection::Borrowed(conn),
96            stream,
97            finished: false,
98        })
99    }
100
101    pub(crate) fn start_owned_with_params(
102        conn: Connection,
103        sql: &str,
104        format: OutputFormat,
105        params: impl Into<QueryParams>,
106    ) -> Result<Self> {
107        let stream = Self::start_query_with_params(conn.handle(), sql, format, params)?;
108        Ok(Self {
109            conn: QueryStreamConnection::Owned(conn),
110            stream,
111            finished: false,
112        })
113    }
114
115    fn start_query(
116        conn: bindings::chdb_connection,
117        sql: &str,
118        format: OutputFormat,
119    ) -> Result<*mut bindings::chdb_result> {
120        let format = format.as_str();
121        // chdb_stream_query_n takes pointer + length for both strings, so
122        // neither has to be NUL-terminated. It returns an owned streaming
123        // chdb_result handle (or null on failure) that the caller must both
124        // cancel with chdb_stream_cancel_query and free with
125        // chdb_destroy_query_result once done — QueryStream::cancel (called
126        // from Drop) does both. `check_start`, below, probes the handle for a
127        // start-up error without taking ownership of it.
128        let stream_ptr = unsafe {
129            bindings::chdb_stream_query_n(
130                conn,
131                sql.as_ptr() as *const c_char,
132                sql.len(),
133                format.as_ptr() as *const c_char,
134                format.len(),
135            )
136        };
137
138        Self::check_start(stream_ptr)
139    }
140
141    /// A non-null stream handle may still carry an initialisation error, so the
142    /// handle is probed once before it is handed out. The probe must not free
143    /// the handle on the success path, hence the `ManuallyDrop` dance.
144    fn check_start(stream_ptr: *mut bindings::chdb_result) -> Result<*mut bindings::chdb_result> {
145        if stream_ptr.is_null() {
146            return Err(Error::NoResult);
147        }
148
149        let probe = ManuallyDrop::new(QueryResult::new(stream_ptr));
150        if let Err(e) = probe.check_error_ref() {
151            drop(ManuallyDrop::into_inner(probe));
152            return Err(e);
153        }
154        std::mem::forget(ManuallyDrop::into_inner(probe));
155
156        Ok(stream_ptr)
157    }
158
159    fn start_query_with_params(
160        conn: bindings::chdb_connection,
161        sql: &str,
162        format: OutputFormat,
163        params: impl Into<QueryParams>,
164    ) -> Result<*mut bindings::chdb_result> {
165        let format = format.as_str();
166        let encoded = EncodedParams::encode(params)?;
167
168        // chdb_stream_query_with_params_n takes pointer + length for query,
169        // format, and each bound value. It returns an owned streaming
170        // chdb_result handle (or null on failure) that QueryStream::cancel
171        // (called from Drop) cancels and frees.
172        let stream_ptr = unsafe {
173            bindings::chdb_stream_query_with_params_n(
174                conn,
175                sql.as_ptr() as *const c_char,
176                sql.len(),
177                format.as_ptr() as *const c_char,
178                format.len(),
179                encoded.names_ptr(),
180                encoded.name_lens_ptr(),
181                encoded.values_ptr(),
182                encoded.value_lens_ptr(),
183                encoded.len(),
184            )
185        };
186
187        Self::check_start(stream_ptr)
188    }
189
190    fn conn_handle(&self) -> bindings::chdb_connection {
191        match &self.conn {
192            QueryStreamConnection::Borrowed(conn) => conn.handle(),
193            QueryStreamConnection::Owned(conn) => conn.handle(),
194        }
195    }
196
197    /// Fetch the next chunk of query results.
198    ///
199    /// Each call returns one chunk of output in the format specified when the stream was
200    /// started. Drop the previous chunk before calling this method again.
201    ///
202    /// # Returns
203    ///
204    /// Returns `Ok(Some(chunk))` while data remains, or `Ok(None)` once the stream is
205    /// exhausted. Each returned chunk is an independent [`QueryResult`] that is freed
206    /// when dropped.
207    ///
208    /// # Examples
209    ///
210    /// ```no_run
211    /// use chdb_rust::connection::Connection;
212    /// use chdb_rust::format::OutputFormat;
213    ///
214    /// let mut conn = Connection::open_in_memory()?;
215    /// let mut stream = conn.query_stream(
216    ///     "SELECT number FROM numbers(10)",
217    ///     OutputFormat::JSONEachRow,
218    /// )?;
219    ///
220    /// while let Some(chunk) = stream.next_chunk()? {
221    ///     println!("rows in chunk: {}", chunk.rows_read());
222    /// }
223    /// # Ok::<(), chdb_rust::error::Error>(())
224    /// ```
225    ///
226    /// # Errors
227    ///
228    /// Returns an error if:
229    /// - The underlying fetch call fails
230    /// - A chunk contains a query error
231    pub fn next_chunk(&mut self) -> Result<Option<QueryResult>> {
232        if self.finished || self.stream.is_null() {
233            return Ok(None);
234        }
235
236        let chunk_ptr =
237            unsafe { bindings::chdb_stream_fetch_result(self.conn_handle(), self.stream) };
238
239        if chunk_ptr.is_null() {
240            self.finished = true;
241            return Err(Error::NoResult);
242        }
243
244        let chunk = QueryResult::new(chunk_ptr);
245
246        if let Err(e) = chunk.check_error_ref() {
247            self.finished = true;
248            return Err(e);
249        }
250
251        if chunk.rows_read() == 0 {
252            drop(chunk);
253            self.finished = true;
254            return Ok(None);
255        }
256
257        Ok(Some(chunk))
258    }
259
260    /// Cancel the streaming query and release resources.
261    ///
262    /// Stops the query on the server and frees the stream handle. After cancellation,
263    /// further calls to [`next_chunk`](Self::next_chunk) return `Ok(None)`.
264    ///
265    /// This is called automatically when the stream is dropped. Call this explicitly to
266    /// stop early without waiting for the stream to be dropped.
267    pub fn cancel(&mut self) {
268        if self.stream.is_null() {
269            return;
270        }
271
272        let conn = self.conn_handle();
273        unsafe {
274            if !self.finished {
275                bindings::chdb_stream_cancel_query(conn, self.stream);
276            }
277            bindings::chdb_destroy_query_result(self.stream);
278        }
279        self.stream = std::ptr::null_mut();
280        self.finished = true;
281    }
282}
283
284impl<'a> Iterator for QueryStream<'a> {
285    type Item = Result<QueryResult>;
286
287    fn next(&mut self) -> Option<Self::Item> {
288        match self.next_chunk() {
289            Ok(Some(chunk)) => Some(Ok(chunk)),
290            Ok(None) => None,
291            Err(err) => Some(Err(err)),
292        }
293    }
294}
295
296impl Drop for QueryStream<'_> {
297    fn drop(&mut self) {
298        if !self.stream.is_null() {
299            self.cancel();
300        }
301    }
302}
303
304#[cfg(test)]
305mod tests {
306    use std::ffi::CString;
307
308    use super::*;
309    use crate::bindings;
310    use crate::session::SessionBuilder;
311    use crate::test_utils::tempdir;
312
313    #[test]
314    fn test_large_stream_row_count_and_chunking() -> Result<()> {
315        const ROWS: u64 = 100_000;
316        let mut conn = Connection::open_in_memory()?;
317        let mut stream = conn.query_stream(
318            &format!("SELECT number FROM numbers({ROWS})"),
319            OutputFormat::JSONEachRow,
320        )?;
321
322        let mut chunks = 0usize;
323        let mut rows = 0u64;
324        while let Some(chunk) = stream.next_chunk()? {
325            chunks += 1;
326            rows += chunk.rows_read();
327        }
328
329        assert_eq!(rows, ROWS, "streamed row count should match query size");
330        assert!(
331            chunks > 1,
332            "expected multiple chunks for large result set, got {chunks}"
333        );
334        Ok(())
335    }
336
337    #[test]
338    fn test_large_stream_matches_materialized_query() -> Result<()> {
339        use crate::arg::Arg;
340        use crate::execute;
341
342        const ROWS: u64 = 100_000;
343        let query = format!("SELECT number FROM numbers({ROWS})");
344        let args = Some(&[Arg::OutputFormat(OutputFormat::JSONEachRow)] as &[Arg]);
345
346        let materialized = execute(&query, args)?;
347        let mut streamed = String::new();
348        let mut conn = Connection::open_in_memory()?;
349        let mut stream = conn.query_stream(&query, OutputFormat::JSONEachRow)?;
350        while let Some(chunk) = stream.next_chunk()? {
351            streamed.push_str(&chunk.data_utf8_lossy());
352        }
353
354        assert_eq!(streamed, materialized.data_utf8_lossy());
355        Ok(())
356    }
357
358    #[test]
359    fn test_raw_stream_ffi_matches_c() -> Result<()> {
360        let conn = Connection::open_in_memory()?;
361        let query = CString::new("SELECT number FROM numbers(5)")?;
362        let format = CString::new("JSONEachRow")?;
363
364        let stream =
365            unsafe { bindings::chdb_stream_query(conn.handle(), query.as_ptr(), format.as_ptr()) };
366        assert!(!stream.is_null());
367
368        let chunk = unsafe { bindings::chdb_stream_fetch_result(conn.handle(), stream) };
369        assert!(!chunk.is_null(), "fetch returned null");
370
371        let result = QueryResult::new(chunk);
372        result.check_error_ref()?;
373        assert_eq!(result.rows_read(), 5);
374        assert!(result.data_utf8_lossy().contains("\"number\":0"));
375
376        unsafe {
377            bindings::chdb_stream_cancel_query(conn.handle(), stream);
378            bindings::chdb_destroy_query_result(stream);
379        }
380        Ok(())
381    }
382
383    #[test]
384    fn test_execute_stream_matches_query() -> Result<()> {
385        use crate::arg::Arg;
386        use crate::execute;
387        use crate::execute_stream;
388
389        let query = "SELECT number FROM numbers(10)";
390        let args = Some(&[Arg::OutputFormat(OutputFormat::JSONEachRow)] as &[Arg]);
391
392        let materialized = execute(query, args)?;
393        let mut streamed = String::new();
394        let mut stream = execute_stream(query, args)?;
395        while let Some(chunk) = stream.next_chunk()? {
396            streamed.push_str(&chunk.data_utf8_lossy());
397        }
398
399        assert_eq!(streamed, materialized.data_utf8_lossy());
400        Ok(())
401    }
402
403    #[test]
404    fn test_query_stream_basic() -> Result<()> {
405        let mut conn = Connection::open_in_memory()?;
406        let mut stream =
407            conn.query_stream("SELECT number FROM numbers(5)", OutputFormat::JSONEachRow)?;
408
409        let mut chunks = Vec::new();
410        while let Some(chunk) = stream.next_chunk()? {
411            chunks.push(chunk.data_utf8_lossy().to_string());
412        }
413
414        let combined: String = chunks.concat();
415        assert!(combined.contains("\"number\":0"));
416        assert!(combined.contains("\"number\":4"));
417        Ok(())
418    }
419
420    #[test]
421    fn test_query_stream_iterator() -> Result<()> {
422        let mut conn = Connection::open_in_memory()?;
423        let stream = conn.query_stream("SELECT 1 AS a UNION ALL SELECT 2", OutputFormat::CSV)?;
424
425        let chunks: Vec<_> = stream.collect::<Result<Vec<_>>>()?;
426        assert!(!chunks.is_empty());
427        Ok(())
428    }
429
430    #[test]
431    fn test_query_stream_error_then_retry_returns_none() -> Result<()> {
432        let mut conn = Connection::open_in_memory()?;
433        let mut stream =
434            conn.query_stream("SELECT * FROM nonexistent_table", OutputFormat::JSONEachRow)?;
435
436        assert!(stream.next_chunk().is_err());
437        assert!(stream.next_chunk()?.is_none());
438        Ok(())
439    }
440
441    #[test]
442    fn test_query_stream_syntax_error_fails_at_start() -> Result<()> {
443        let mut conn = Connection::open_in_memory()?;
444        let result = conn.query_stream("SELECT invalid syntax here", OutputFormat::JSONEachRow);
445        assert!(result.is_err());
446        Ok(())
447    }
448
449    #[test]
450    fn test_query_stream_with_params_error_then_retry_returns_none() -> Result<()> {
451        let mut conn = Connection::open_in_memory()?;
452        let mut stream = conn.query_stream_with_params(
453            "SELECT * FROM nonexistent_table WHERE id = {id:UInt64}",
454            OutputFormat::JSONEachRow,
455            [("id", 1_u64)],
456        )?;
457
458        assert!(stream.next_chunk().is_err());
459        assert!(stream.next_chunk()?.is_none());
460        Ok(())
461    }
462
463    #[test]
464    fn test_query_stream_with_params_syntax_error_fails_at_start() -> Result<()> {
465        let mut conn = Connection::open_in_memory()?;
466        let result = conn.query_stream_with_params(
467            "SELECT invalid syntax here",
468            OutputFormat::JSONEachRow,
469            [("x", 1_u64)],
470        );
471        assert!(result.is_err());
472        Ok(())
473    }
474
475    #[test]
476    fn test_session_execute_stream() -> Result<()> {
477        let tmp = tempdir();
478        let mut session = SessionBuilder::new()
479            .with_data_path(tmp.path())
480            .with_auto_cleanup(true)
481            .build()?;
482
483        session.execute(
484            "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
485            None,
486        )?;
487        session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
488
489        let mut stream = session.execute_stream("SELECT * FROM items", None)?;
490        let mut total_rows = 0usize;
491        while let Some(chunk) = stream.next_chunk()? {
492            total_rows += chunk.data_ref().iter().filter(|&&b| b == b'\n').count();
493        }
494
495        assert_eq!(total_rows, 3);
496        Ok(())
497    }
498}