1use std::ffi::CStr;
10use std::mem::ManuallyDrop;
11use std::os::raw::c_char;
12
13use arrow::array::RecordBatchReader;
14use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
15use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
16use arrow::record_batch::RecordBatch;
17
18use crate::arrow_options::ArrowOptions;
19use crate::bindings;
20use crate::connection::Connection;
21use crate::error::{Error, Result};
22use crate::query_param::{EncodedParams, QueryParams};
23use crate::query_result::QueryResult;
24
25enum ArrowQueryStreamConnection<'a> {
26 Borrowed(&'a mut Connection),
27 Owned(Connection),
28}
29
30pub struct ArrowReader {
40 inner: ArrowArrayStreamReader,
41}
42
43impl ArrowReader {
44 pub(crate) fn new(inner: ArrowArrayStreamReader) -> Self {
45 Self { inner }
46 }
47}
48
49impl Iterator for ArrowReader {
50 type Item = arrow::error::Result<RecordBatch>;
51
52 fn next(&mut self) -> Option<Self::Item> {
53 self.inner.next()
54 }
55}
56
57impl arrow::array::RecordBatchReader for ArrowReader {
58 fn schema(&self) -> arrow::datatypes::SchemaRef {
59 self.inner.schema()
60 }
61}
62
63pub struct ArrowQueryStream<'a> {
76 conn: ArrowQueryStreamConnection<'a>,
77 inner: *mut bindings::chdb_result,
78 finished: bool,
79 schema: Option<arrow::datatypes::SchemaRef>,
80}
81
82impl<'a> ArrowQueryStream<'a> {
83 pub(crate) fn start_borrowed(
84 conn: &'a mut Connection,
85 sql: &str,
86 opts: Option<&ArrowOptions>,
87 ) -> Result<Self> {
88 let inner = Self::start_query(conn.handle(), sql, opts)?;
89 Ok(Self {
90 conn: ArrowQueryStreamConnection::Borrowed(conn),
91 inner,
92 finished: false,
93 schema: None,
94 })
95 }
96
97 pub(crate) fn start_owned(
98 conn: Connection,
99 sql: &str,
100 opts: Option<&ArrowOptions>,
101 ) -> Result<Self> {
102 let inner = Self::start_query(conn.handle(), sql, opts)?;
103 Ok(Self {
104 conn: ArrowQueryStreamConnection::Owned(conn),
105 inner,
106 finished: false,
107 schema: None,
108 })
109 }
110
111 pub(crate) fn start_borrowed_with_params(
112 conn: &'a mut Connection,
113 sql: &str,
114 params: impl Into<QueryParams>,
115 opts: Option<&ArrowOptions>,
116 ) -> Result<Self> {
117 let inner = Self::start_query_with_params(conn.handle(), sql, params, opts)?;
118 Ok(Self {
119 conn: ArrowQueryStreamConnection::Borrowed(conn),
120 inner,
121 finished: false,
122 schema: None,
123 })
124 }
125
126 pub(crate) fn start_owned_with_params(
127 conn: Connection,
128 sql: &str,
129 params: impl Into<QueryParams>,
130 opts: Option<&ArrowOptions>,
131 ) -> Result<Self> {
132 let inner = Self::start_query_with_params(conn.handle(), sql, params, opts)?;
133 Ok(Self {
134 conn: ArrowQueryStreamConnection::Owned(conn),
135 inner,
136 finished: false,
137 schema: None,
138 })
139 }
140
141 fn start_query(
142 conn: bindings::chdb_connection,
143 sql: &str,
144 opts: Option<&ArrowOptions>,
145 ) -> Result<*mut bindings::chdb_result> {
146 let c_opts = opts.map(|o| o.to_c());
157 let opts_ptr = c_opts.as_ref().map_or(std::ptr::null(), |o| {
158 o as *const bindings::chdb_arrow_options
159 });
160
161 let stream_ptr = unsafe {
162 bindings::chdb_stream_query_arrow_n(
163 conn,
164 sql.as_ptr() as *const c_char,
165 sql.len(),
166 opts_ptr,
167 )
168 };
169
170 Self::check_start(stream_ptr)
171 }
172
173 fn check_start(stream_ptr: *mut bindings::chdb_result) -> Result<*mut bindings::chdb_result> {
183 if stream_ptr.is_null() {
184 return Err(Error::NoResult);
185 }
186
187 let probe = ManuallyDrop::new(QueryResult::new(stream_ptr));
188 if let Err(e) = probe.check_error_ref() {
189 drop(ManuallyDrop::into_inner(probe));
190 return Err(e);
191 }
192 std::mem::forget(ManuallyDrop::into_inner(probe));
193
194 Ok(stream_ptr)
195 }
196
197 fn start_query_with_params(
198 conn: bindings::chdb_connection,
199 sql: &str,
200 params: impl Into<QueryParams>,
201 opts: Option<&ArrowOptions>,
202 ) -> Result<*mut bindings::chdb_result> {
203 let encoded = EncodedParams::encode(params)?;
204 let c_opts = opts.map(|o| o.to_c());
206 let opts_ptr = c_opts.as_ref().map_or(std::ptr::null(), |o| {
207 o as *const bindings::chdb_arrow_options
208 });
209
210 let stream_ptr = unsafe {
216 bindings::chdb_stream_query_arrow_with_params_n(
217 conn,
218 sql.as_ptr() as *const c_char,
219 sql.len(),
220 opts_ptr,
221 encoded.names_ptr(),
222 encoded.name_lens_ptr(),
223 encoded.values_ptr(),
224 encoded.value_lens_ptr(),
225 encoded.len(),
226 )
227 };
228
229 Self::check_start(stream_ptr)
230 }
231
232 fn conn_handle(&self) -> bindings::chdb_connection {
233 match &self.conn {
234 ArrowQueryStreamConnection::Borrowed(conn) => conn.handle(),
235 ArrowQueryStreamConnection::Owned(conn) => conn.handle(),
236 }
237 }
238
239 pub fn schema(&self) -> Option<arrow::datatypes::SchemaRef> {
244 self.schema.clone()
245 }
246
247 pub fn next_batch(&mut self) -> Result<Option<RecordBatch>> {
249 if self.finished || self.inner.is_null() {
250 return Ok(None);
251 }
252
253 let mut ffi_stream = FFI_ArrowArrayStream::empty();
254 let out_ptr = (&mut ffi_stream as *mut FFI_ArrowArrayStream).cast();
255
256 let state =
257 unsafe { bindings::chdb_stream_fetch_arrow(self.conn_handle(), self.inner, out_ptr) };
258
259 if state != bindings::chdb_state_CHDBSuccess {
260 let err_msg = unsafe { bindings::chdb_result_error(self.inner) };
261 let detail = if err_msg.is_null() {
262 "chdb_stream_fetch_arrow failed".to_string()
263 } else {
264 unsafe { CStr::from_ptr(err_msg) }
265 .to_string_lossy()
266 .into_owned()
267 };
268 self.cancel();
269 return Err(Error::QueryError(detail));
270 }
271
272 match is_eos_without_schema(&mut ffi_stream) {
273 Ok(true) => {
274 self.finished = true;
275 return Ok(None);
276 }
277 Ok(false) => {}
278 Err(err) => {
279 self.cancel();
280 return Err(err);
281 }
282 }
283
284 let mut reader = match ArrowArrayStreamReader::try_new(ffi_stream) {
285 Ok(reader) => reader,
286 Err(e) => {
287 self.cancel();
288 return Err(Error::QueryError(e.to_string()));
289 }
290 };
291
292 self.schema = Some(reader.schema());
293
294 match reader.next() {
295 Some(Ok(batch)) => Ok(Some(batch)),
296 Some(Err(e)) => {
297 self.cancel();
298 Err(Error::QueryError(e.to_string()))
299 }
300 None => {
301 self.finished = true;
302 Ok(None)
303 }
304 }
305 }
306
307 pub fn cancel(&mut self) {
309 if self.inner.is_null() {
310 return;
311 }
312
313 if !self.finished {
314 unsafe {
315 bindings::chdb_stream_cancel_query(self.conn_handle(), self.inner);
316 }
317 }
318 unsafe {
319 bindings::chdb_destroy_query_result(self.inner);
320 }
321 self.inner = std::ptr::null_mut();
322 self.finished = true;
323 }
324}
325
326fn is_eos_without_schema(ffi_stream: &mut FFI_ArrowArrayStream) -> Result<bool> {
330 let Some(get_schema) = ffi_stream.get_schema else {
331 return Err(Error::InvalidData(
332 "Arrow stream has no get_schema callback".into(),
333 ));
334 };
335 let mut schema = FFI_ArrowSchema::empty();
336 let ret_code = unsafe { get_schema(ffi_stream, &mut schema) };
337 if ret_code == 0 {
338 return Ok(false);
339 }
340
341 let Some(get_next) = ffi_stream.get_next else {
342 return Err(Error::InvalidData(
343 "Arrow stream has no get_next callback".into(),
344 ));
345 };
346 let mut array = FFI_ArrowArray::empty();
347 let next_code = unsafe { get_next(ffi_stream, &mut array) };
348 let detail = ffi_stream.get_last_error.and_then(|get_error| {
349 let ptr = unsafe { get_error(ffi_stream) };
350 (!ptr.is_null()).then(|| {
351 unsafe { CStr::from_ptr(ptr) }
352 .to_string_lossy()
353 .into_owned()
354 })
355 });
356 if next_code == 0 && array.release.is_none() && detail.is_none() {
357 return Ok(true);
358 }
359 Err(Error::QueryError(detail.unwrap_or_else(|| {
360 format!("Arrow stream schema failed ({ret_code}); next batch returned {next_code}")
361 })))
362}
363
364impl Iterator for ArrowQueryStream<'_> {
365 type Item = Result<RecordBatch>;
366
367 fn next(&mut self) -> Option<Self::Item> {
368 match self.next_batch() {
369 Ok(Some(batch)) => Some(Ok(batch)),
370 Ok(None) => None,
371 Err(err) => Some(Err(err)),
372 }
373 }
374}
375
376impl Drop for ArrowQueryStream<'_> {
377 fn drop(&mut self) {
378 if !self.inner.is_null() {
379 self.cancel();
380 }
381 }
382}
383
384#[cfg(test)]
385mod tests {
386 use super::*;
387 use crate::session::SessionBuilder;
388 use crate::test_utils::tempdir;
389
390 unsafe extern "C" fn failing_schema(
391 _: *mut FFI_ArrowArrayStream,
392 _: *mut FFI_ArrowSchema,
393 ) -> i32 {
394 1
395 }
396
397 unsafe extern "C" fn empty_next(_: *mut FFI_ArrowArrayStream, _: *mut FFI_ArrowArray) -> i32 {
398 0
399 }
400
401 unsafe extern "C" fn schema_error(_: *mut FFI_ArrowArrayStream) -> *const c_char {
402 c"invalid schema".as_ptr()
403 }
404
405 #[test]
406 fn schema_error_is_not_silently_treated_as_eof() {
407 let mut stream = FFI_ArrowArrayStream::empty();
408 stream.get_schema = Some(failing_schema);
409 stream.get_next = Some(empty_next);
410 stream.get_last_error = Some(schema_error);
411 let err = is_eos_without_schema(&mut stream).unwrap_err();
412 assert!(err.to_string().contains("invalid schema"));
413
414 stream.get_last_error = None;
415 assert!(is_eos_without_schema(&mut stream).unwrap());
416 }
417
418 #[test]
419 fn test_arrow_query_stream_row_count_and_chunking() -> Result<()> {
420 let mut conn = Connection::open_in_memory()?;
421 let mut stream = conn.query_stream_arrow("SELECT number FROM numbers(100_000)")?;
422
423 let mut batches = 0usize;
424 let mut rows = 0usize;
425 while let Some(batch) = stream.next_batch()? {
426 batches += 1;
427 rows += batch.num_rows();
428 }
429
430 assert_eq!(rows, 100_000);
431 assert!(batches > 1, "expected multiple batches, got {batches}");
432 Ok(())
433 }
434
435 #[test]
436 fn test_execute_stream_arrow() -> Result<()> {
437 use crate::execute_stream_arrow;
438
439 let mut stream = execute_stream_arrow("SELECT number FROM numbers(10)")?;
440 let mut rows = 0usize;
441 while let Some(batch) = stream.next_batch()? {
442 rows += batch.num_rows();
443 }
444
445 assert_eq!(rows, 10);
446 Ok(())
447 }
448
449 #[test]
450 fn test_session_execute_stream_arrow() -> Result<()> {
451 let tmp = tempdir();
452 let mut session = SessionBuilder::new()
453 .with_data_path(tmp.path())
454 .with_auto_cleanup(true)
455 .build()?;
456
457 session.execute(
458 "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
459 None,
460 )?;
461 session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
462
463 let mut stream = session.execute_stream_arrow("SELECT * FROM items")?;
464 let mut rows = 0usize;
465 while let Some(batch) = stream.next_batch()? {
466 rows += batch.num_rows();
467 }
468
469 assert_eq!(rows, 3);
470 Ok(())
471 }
472
473 #[test]
474 fn test_arrow_query_stream_iterator() -> Result<()> {
475 let mut conn = Connection::open_in_memory()?;
476 let stream = conn.query_stream_arrow("SELECT number FROM numbers(5)")?;
477
478 let batches: Vec<_> = stream.collect::<Result<Vec<_>>>()?;
479 assert_eq!(batches.len(), 1);
480 assert_eq!(batches[0].num_rows(), 5);
481 Ok(())
482 }
483
484 #[test]
485 fn test_arrow_query_stream_empty_filter_returns_none() -> Result<()> {
486 let tmp = tempdir();
487 let mut session = SessionBuilder::new()
488 .with_data_path(tmp.path())
489 .with_auto_cleanup(true)
490 .build()?;
491
492 session.execute(
493 "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
494 None,
495 )?;
496 session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
497
498 let mut stream = session.execute_stream_arrow("SELECT * FROM items WHERE id > 100")?;
499 assert!(stream.next_batch()?.is_none());
500 assert!(stream.next_batch()?.is_none());
501 Ok(())
502 }
503
504 #[test]
505 fn test_arrow_query_stream_error_then_retry_returns_none() -> Result<()> {
506 let mut conn = Connection::open_in_memory()?;
507 let mut stream = conn.query_stream_arrow("SELECT * FROM nonexistent_table")?;
508
509 assert!(stream.next_batch().is_err());
510 assert!(stream.next_batch()?.is_none());
511 Ok(())
512 }
513
514 #[test]
515 fn test_arrow_query_stream_syntax_error_fails_at_start() -> Result<()> {
516 let mut conn = Connection::open_in_memory()?;
517 let result = conn.query_stream_arrow("SELECT invalid syntax here");
518 assert!(result.is_err());
519 Ok(())
520 }
521
522 #[test]
523 fn test_arrow_query_stream_with_params_empty_filter_returns_none() -> Result<()> {
524 let tmp = tempdir();
525 let mut session = SessionBuilder::new()
526 .with_data_path(tmp.path())
527 .with_auto_cleanup(true)
528 .build()?;
529
530 session.execute(
531 "CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
532 None,
533 )?;
534 session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
535
536 let mut stream = session.execute_stream_arrow_with_params(
537 "SELECT * FROM items WHERE id > {min_id:UInt64}",
538 [("min_id", 100_u64)],
539 None,
540 )?;
541 assert!(stream.next_batch()?.is_none());
542 assert!(stream.next_batch()?.is_none());
543 Ok(())
544 }
545
546 #[test]
547 fn test_arrow_query_stream_with_params_error_then_retry_returns_none() -> Result<()> {
548 let mut conn = Connection::open_in_memory()?;
549 let mut stream = conn.query_stream_arrow_with_params(
550 "SELECT * FROM nonexistent_table WHERE id = {id:UInt64}",
551 [("id", 1_u64)],
552 None,
553 )?;
554
555 assert!(stream.next_batch().is_err());
556 assert!(stream.next_batch()?.is_none());
557 Ok(())
558 }
559
560 #[test]
561 fn test_arrow_query_stream_with_params_syntax_error_fails_at_start() -> Result<()> {
562 let mut conn = Connection::open_in_memory()?;
563 let result =
564 conn.query_stream_arrow_with_params("SELECT invalid syntax here", [("x", 1_u64)], None);
565 assert!(result.is_err());
566 Ok(())
567 }
568
569 #[test]
570 fn test_arrow_query_stream_early_drop() -> Result<()> {
571 let mut conn = Connection::open_in_memory()?;
572 let mut stream = conn.query_stream_arrow("SELECT number FROM numbers(1_000_000)")?;
573 let first = stream.next_batch()?.expect("expected a batch");
574 assert!(first.num_rows() > 0);
575 drop(stream);
576 Ok(())
577 }
578}