1use std::mem::ManuallyDrop;
7use std::os::raw::c_char;
8
9use crate::bindings;
10use crate::connection::Connection;
11use crate::error::{Error, Result};
12use crate::format::OutputFormat;
13use crate::query_param::{EncodedParams, QueryParams};
14use crate::query_result::QueryResult;
15
16enum QueryStreamConnection<'a> {
17 Borrowed(&'a mut Connection),
18 Owned(Connection),
19}
20
21pub struct QueryStream<'a> {
59 conn: QueryStreamConnection<'a>,
60 stream: *mut bindings::chdb_result,
61 finished: bool,
62}
63
64impl<'a> QueryStream<'a> {
65 pub(crate) fn start_borrowed(
66 conn: &'a mut Connection,
67 sql: &str,
68 format: OutputFormat,
69 ) -> Result<Self> {
70 let stream = Self::start_query(conn.handle(), sql, format)?;
71 Ok(Self {
72 conn: QueryStreamConnection::Borrowed(conn),
73 stream,
74 finished: false,
75 })
76 }
77
78 pub(crate) fn start_owned(conn: Connection, sql: &str, format: OutputFormat) -> Result<Self> {
79 let stream = Self::start_query(conn.handle(), sql, format)?;
80 Ok(Self {
81 conn: QueryStreamConnection::Owned(conn),
82 stream,
83 finished: false,
84 })
85 }
86
87 pub(crate) fn start_borrowed_with_params(
88 conn: &'a mut Connection,
89 sql: &str,
90 format: OutputFormat,
91 params: impl Into<QueryParams>,
92 ) -> Result<Self> {
93 let stream = Self::start_query_with_params(conn.handle(), sql, format, params)?;
94 Ok(Self {
95 conn: QueryStreamConnection::Borrowed(conn),
96 stream,
97 finished: false,
98 })
99 }
100
101 pub(crate) fn start_owned_with_params(
102 conn: Connection,
103 sql: &str,
104 format: OutputFormat,
105 params: impl Into<QueryParams>,
106 ) -> Result<Self> {
107 let stream = Self::start_query_with_params(conn.handle(), sql, format, params)?;
108 Ok(Self {
109 conn: QueryStreamConnection::Owned(conn),
110 stream,
111 finished: false,
112 })
113 }
114
115 fn start_query(
116 conn: bindings::chdb_connection,
117 sql: &str,
118 format: OutputFormat,
119 ) -> Result<*mut bindings::chdb_result> {
120 let format = format.as_str();
121 let stream_ptr = unsafe {
129 bindings::chdb_stream_query_n(
130 conn,
131 sql.as_ptr() as *const c_char,
132 sql.len(),
133 format.as_ptr() as *const c_char,
134 format.len(),
135 )
136 };
137
138 Self::check_start(stream_ptr)
139 }
140
141 fn check_start(stream_ptr: *mut bindings::chdb_result) -> Result<*mut bindings::chdb_result> {
145 if stream_ptr.is_null() {
146 return Err(Error::NoResult);
147 }
148
149 let probe = ManuallyDrop::new(QueryResult::new(stream_ptr));
150 if let Err(e) = probe.check_error_ref() {
151 drop(ManuallyDrop::into_inner(probe));
152 return Err(e);
153 }
154 std::mem::forget(ManuallyDrop::into_inner(probe));
155
156 Ok(stream_ptr)
157 }
158
159 fn start_query_with_params(
160 conn: bindings::chdb_connection,
161 sql: &str,
162 format: OutputFormat,
163 params: impl Into<QueryParams>,
164 ) -> Result<*mut bindings::chdb_result> {
165 let format = format.as_str();
166 let encoded = EncodedParams::encode(params)?;
167
168 let stream_ptr = unsafe {
173 bindings::chdb_stream_query_with_params_n(
174 conn,
175 sql.as_ptr() as *const c_char,
176 sql.len(),
177 format.as_ptr() as *const c_char,
178 format.len(),
179 encoded.names_ptr(),
180 encoded.name_lens_ptr(),
181 encoded.values_ptr(),
182 encoded.value_lens_ptr(),
183 encoded.len(),
184 )
185 };
186
187 Self::check_start(stream_ptr)
188 }
189
190 fn conn_handle(&self) -> bindings::chdb_connection {
191 match &self.conn {
192 QueryStreamConnection::Borrowed(conn) => conn.handle(),
193 QueryStreamConnection::Owned(conn) => conn.handle(),
194 }
195 }
196
197 pub fn next_chunk(&mut self) -> Result<Option<QueryResult>> {
232 if self.finished || self.stream.is_null() {
233 return Ok(None);
234 }
235
236 let chunk_ptr =
237 unsafe { bindings::chdb_stream_fetch_result(self.conn_handle(), self.stream) };
238
239 if chunk_ptr.is_null() {
240 self.finished = true;
241 return Err(Error::NoResult);
242 }
243
244 let chunk = QueryResult::new(chunk_ptr);
245
246 if let Err(e) = chunk.check_error_ref() {
247 self.finished = true;
248 return Err(e);
249 }
250
251 if chunk.rows_read() == 0 {
252 drop(chunk);
253 self.finished = true;
254 return Ok(None);
255 }
256
257 Ok(Some(chunk))
258 }
259
260 pub fn cancel(&mut self) {
268 if self.stream.is_null() {
269 return;
270 }
271
272 let conn = self.conn_handle();
273 unsafe {
274 if !self.finished {
275 bindings::chdb_stream_cancel_query(conn, self.stream);
276 }
277 bindings::chdb_destroy_query_result(self.stream);
278 }
279 self.stream = std::ptr::null_mut();
280 self.finished = true;
281 }
282}
283
284impl<'a> Iterator for QueryStream<'a> {
285 type Item = Result<QueryResult>;
286
287 fn next(&mut self) -> Option<Self::Item> {
288 match self.next_chunk() {
289 Ok(Some(chunk)) => Some(Ok(chunk)),
290 Ok(None) => None,
291 Err(err) => Some(Err(err)),
292 }
293 }
294}
295
296impl Drop for QueryStream<'_> {
297 fn drop(&mut self) {
298 if !self.stream.is_null() {
299 self.cancel();
300 }
301 }
302}
303
304#[cfg(test)]
305mod tests {
306 use std::ffi::CString;
307
308 use super::*;
309 use crate::bindings;
310 use crate::session::SessionBuilder;
311 use crate::test_utils::tempdir;
312
313 #[test]
314 fn test_large_stream_row_count_and_chunking() -> Result<()> {
315 const ROWS: u64 = 100_000;
316 let mut conn = Connection::open_in_memory()?;
317 let mut stream = conn.query_stream(
318 &format!("SELECT number FROM numbers({ROWS})"),
319 OutputFormat::JSONEachRow,
320 )?;
321
322 let mut chunks = 0usize;
323 let mut rows = 0u64;
324 while let Some(chunk) = stream.next_chunk()? {
325 chunks += 1;
326 rows += chunk.rows_read();
327 }
328
329 assert_eq!(rows, ROWS, "streamed row count should match query size");
330 assert!(
331 chunks > 1,
332 "expected multiple chunks for large result set, got {chunks}"
333 );
334 Ok(())
335 }
336
337 #[test]
338 fn test_large_stream_matches_materialized_query() -> Result<()> {
339 use crate::arg::Arg;
340 use crate::execute;
341
342 const ROWS: u64 = 100_000;
343 let query = format!("SELECT number FROM numbers({ROWS})");
344 let args = Some(&[Arg::OutputFormat(OutputFormat::JSONEachRow)] as &[Arg]);
345
346 let materialized = execute(&query, args)?;
347 let mut streamed = String::new();
348 let mut conn = Connection::open_in_memory()?;
349 let mut stream = conn.query_stream(&query, OutputFormat::JSONEachRow)?;
350 while let Some(chunk) = stream.next_chunk()? {
351 streamed.push_str(&chunk.data_utf8_lossy());
352 }
353
354 assert_eq!(streamed, materialized.data_utf8_lossy());
355 Ok(())
356 }
357
358 #[test]
359 fn test_raw_stream_ffi_matches_c() -> Result<()> {
360 let conn = Connection::open_in_memory()?;
361 let query = CString::new("SELECT number FROM numbers(5)")?;
362 let format = CString::new("JSONEachRow")?;
363
364 let stream =
365 unsafe { bindings::chdb_stream_query(conn.handle(), query.as_ptr(), format.as_ptr()) };
366 assert!(!stream.is_null());
367
368 let chunk = unsafe { bindings::chdb_stream_fetch_result(conn.handle(), stream) };
369 assert!(!chunk.is_null(), "fetch returned null");
370
371 let result = QueryResult::new(chunk);
372 result.check_error_ref()?;
373 assert_eq!(result.rows_read(), 5);
374 assert!(result.data_utf8_lossy().contains("\"number\":0"));
375
376 unsafe {
377 bindings::chdb_stream_cancel_query(conn.handle(), stream);
378 bindings::chdb_destroy_query_result(stream);
379 }
380 Ok(())
381 }
382
383 #[test]
384 fn test_execute_stream_matches_query() -> Result<()> {
385 use crate::arg::Arg;
386 use crate::execute;
387 use crate::execute_stream;
388
389 let query = "SELECT number FROM numbers(10)";
390 let args = Some(&[Arg::OutputFormat(OutputFormat::JSONEachRow)] as &[Arg]);
391
392 let materialized = execute(query, args)?;
393 let mut streamed = String::new();
394 let mut stream = execute_stream(query, args)?;
395 while let Some(chunk) = stream.next_chunk()? {
396 streamed.push_str(&chunk.data_utf8_lossy());
397 }
398
399 assert_eq!(streamed, materialized.data_utf8_lossy());
400 Ok(())
401 }
402
403 #[test]
404 fn test_query_stream_basic() -> Result<()> {
405 let mut conn = Connection::open_in_memory()?;
406 let mut stream =
407 conn.query_stream("SELECT number FROM numbers(5)", OutputFormat::JSONEachRow)?;
408
409 let mut chunks = Vec::new();
410 while let Some(chunk) = stream.next_chunk()? {
411 chunks.push(chunk.data_utf8_lossy().to_string());
412 }
413
414 let combined: String = chunks.concat();
415 assert!(combined.contains("\"number\":0"));
416 assert!(combined.contains("\"number\":4"));
417 Ok(())
418 }
419
420 #[test]
421 fn test_query_stream_iterator() -> Result<()> {
422 let mut conn = Connection::open_in_memory()?;
423 let stream = conn.query_stream("SELECT 1 AS a UNION ALL SELECT 2", OutputFormat::CSV)?;
424
425 let chunks: Vec<_> = stream.collect::<Result<Vec<_>>>()?;
426 assert!(!chunks.is_empty());
427 Ok(())
428 }
429
430 #[test]
431 fn test_query_stream_error_then_retry_returns_none() -> Result<()> {
432 let mut conn = Connection::open_in_memory()?;
433 let mut stream =
434 conn.query_stream("SELECT * FROM nonexistent_table", OutputFormat::JSONEachRow)?;
435
436 assert!(stream.next_chunk().is_err());
437 assert!(stream.next_chunk()?.is_none());
438 Ok(())
439 }
440
441 #[test]
442 fn test_query_stream_syntax_error_fails_at_start() -> Result<()> {
443 let mut conn = Connection::open_in_memory()?;
444 let result = conn.query_stream("SELECT invalid syntax here", OutputFormat::JSONEachRow);
445 assert!(result.is_err());
446 Ok(())
447 }
448
449 #[test]
450 fn test_query_stream_with_params_error_then_retry_returns_none() -> Result<()> {
451 let mut conn = Connection::open_in_memory()?;
452 let mut stream = conn.query_stream_with_params(
453 "SELECT * FROM nonexistent_table WHERE id = {id:UInt64}",
454 OutputFormat::JSONEachRow,
455 [("id", 1_u64)],
456 )?;
457
458 assert!(stream.next_chunk().is_err());
459 assert!(stream.next_chunk()?.is_none());
460 Ok(())
461 }
462
463 #[test]
464 fn test_query_stream_with_params_syntax_error_fails_at_start() -> Result<()> {
465 let mut conn = Connection::open_in_memory()?;
466 let result = conn.query_stream_with_params(
467 "SELECT invalid syntax here",
468 OutputFormat::JSONEachRow,
469 [("x", 1_u64)],
470 );
471 assert!(result.is_err());
472 Ok(())
473 }
474
475 #[test]
476 fn test_session_execute_stream() -> Result<()> {
477 let tmp = tempdir();
478 let mut session = SessionBuilder::new()
479 .with_data_path(tmp.path())
480 .with_auto_cleanup(true)
481 .build()?;
482
483 session.execute(
484 "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
485 None,
486 )?;
487 session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
488
489 let mut stream = session.execute_stream("SELECT * FROM items", None)?;
490 let mut total_rows = 0usize;
491 while let Some(chunk) = stream.next_chunk()? {
492 total_rows += chunk.data_ref().iter().filter(|&&b| b == b'\n').count();
493 }
494
495 assert_eq!(total_rows, 3);
496 Ok(())
497 }
498}