use std::ffi::CStr;
use std::mem::ManuallyDrop;
use std::os::raw::c_char;
use arrow::array::RecordBatchReader;
use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema};
use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
use arrow::record_batch::RecordBatch;
use crate::arrow_options::ArrowOptions;
use crate::bindings;
use crate::connection::Connection;
use crate::error::{Error, Result};
use crate::query_param::{EncodedParams, QueryParams};
use crate::query_result::QueryResult;
enum ArrowQueryStreamConnection<'a> {
Borrowed(&'a mut Connection),
Owned(Connection),
}
pub struct ArrowReader {
inner: ArrowArrayStreamReader,
}
impl ArrowReader {
pub(crate) fn new(inner: ArrowArrayStreamReader) -> Self {
Self { inner }
}
}
impl Iterator for ArrowReader {
type Item = arrow::error::Result<RecordBatch>;
fn next(&mut self) -> Option<Self::Item> {
self.inner.next()
}
}
impl arrow::array::RecordBatchReader for ArrowReader {
fn schema(&self) -> arrow::datatypes::SchemaRef {
self.inner.schema()
}
}
pub struct ArrowQueryStream<'a> {
conn: ArrowQueryStreamConnection<'a>,
inner: *mut bindings::chdb_result,
finished: bool,
schema: Option<arrow::datatypes::SchemaRef>,
}
impl<'a> ArrowQueryStream<'a> {
pub(crate) fn start_borrowed(
conn: &'a mut Connection,
sql: &str,
opts: Option<&ArrowOptions>,
) -> Result<Self> {
let inner = Self::start_query(conn.handle(), sql, opts)?;
Ok(Self {
conn: ArrowQueryStreamConnection::Borrowed(conn),
inner,
finished: false,
schema: None,
})
}
pub(crate) fn start_owned(
conn: Connection,
sql: &str,
opts: Option<&ArrowOptions>,
) -> Result<Self> {
let inner = Self::start_query(conn.handle(), sql, opts)?;
Ok(Self {
conn: ArrowQueryStreamConnection::Owned(conn),
inner,
finished: false,
schema: None,
})
}
pub(crate) fn start_borrowed_with_params(
conn: &'a mut Connection,
sql: &str,
params: impl Into<QueryParams>,
opts: Option<&ArrowOptions>,
) -> Result<Self> {
let inner = Self::start_query_with_params(conn.handle(), sql, params, opts)?;
Ok(Self {
conn: ArrowQueryStreamConnection::Borrowed(conn),
inner,
finished: false,
schema: None,
})
}
pub(crate) fn start_owned_with_params(
conn: Connection,
sql: &str,
params: impl Into<QueryParams>,
opts: Option<&ArrowOptions>,
) -> Result<Self> {
let inner = Self::start_query_with_params(conn.handle(), sql, params, opts)?;
Ok(Self {
conn: ArrowQueryStreamConnection::Owned(conn),
inner,
finished: false,
schema: None,
})
}
fn start_query(
conn: bindings::chdb_connection,
sql: &str,
opts: Option<&ArrowOptions>,
) -> Result<*mut bindings::chdb_result> {
let c_opts = opts.map(|o| o.to_c());
let opts_ptr = c_opts.as_ref().map_or(std::ptr::null(), |o| {
o as *const bindings::chdb_arrow_options
});
let stream_ptr = unsafe {
bindings::chdb_stream_query_arrow_n(
conn,
sql.as_ptr() as *const c_char,
sql.len(),
opts_ptr,
)
};
Self::check_start(stream_ptr)
}
fn check_start(stream_ptr: *mut bindings::chdb_result) -> Result<*mut bindings::chdb_result> {
if stream_ptr.is_null() {
return Err(Error::NoResult);
}
let probe = ManuallyDrop::new(QueryResult::new(stream_ptr));
if let Err(e) = probe.check_error_ref() {
drop(ManuallyDrop::into_inner(probe));
return Err(e);
}
std::mem::forget(ManuallyDrop::into_inner(probe));
Ok(stream_ptr)
}
fn start_query_with_params(
conn: bindings::chdb_connection,
sql: &str,
params: impl Into<QueryParams>,
opts: Option<&ArrowOptions>,
) -> Result<*mut bindings::chdb_result> {
let encoded = EncodedParams::encode(params)?;
let c_opts = opts.map(|o| o.to_c());
let opts_ptr = c_opts.as_ref().map_or(std::ptr::null(), |o| {
o as *const bindings::chdb_arrow_options
});
let stream_ptr = unsafe {
bindings::chdb_stream_query_arrow_with_params_n(
conn,
sql.as_ptr() as *const c_char,
sql.len(),
opts_ptr,
encoded.names_ptr(),
encoded.name_lens_ptr(),
encoded.values_ptr(),
encoded.value_lens_ptr(),
encoded.len(),
)
};
Self::check_start(stream_ptr)
}
fn conn_handle(&self) -> bindings::chdb_connection {
match &self.conn {
ArrowQueryStreamConnection::Borrowed(conn) => conn.handle(),
ArrowQueryStreamConnection::Owned(conn) => conn.handle(),
}
}
pub fn schema(&self) -> Option<arrow::datatypes::SchemaRef> {
self.schema.clone()
}
pub fn next_batch(&mut self) -> Result<Option<RecordBatch>> {
if self.finished || self.inner.is_null() {
return Ok(None);
}
let mut ffi_stream = FFI_ArrowArrayStream::empty();
let out_ptr = (&mut ffi_stream as *mut FFI_ArrowArrayStream).cast();
let state =
unsafe { bindings::chdb_stream_fetch_arrow(self.conn_handle(), self.inner, out_ptr) };
if state != bindings::chdb_state_CHDBSuccess {
let err_msg = unsafe { bindings::chdb_result_error(self.inner) };
let detail = if err_msg.is_null() {
"chdb_stream_fetch_arrow failed".to_string()
} else {
unsafe { CStr::from_ptr(err_msg) }
.to_string_lossy()
.into_owned()
};
self.cancel();
return Err(Error::QueryError(detail));
}
match is_eos_without_schema(&mut ffi_stream) {
Ok(true) => {
self.finished = true;
return Ok(None);
}
Ok(false) => {}
Err(err) => {
self.cancel();
return Err(err);
}
}
let mut reader = match ArrowArrayStreamReader::try_new(ffi_stream) {
Ok(reader) => reader,
Err(e) => {
self.cancel();
return Err(Error::QueryError(e.to_string()));
}
};
self.schema = Some(reader.schema());
match reader.next() {
Some(Ok(batch)) => Ok(Some(batch)),
Some(Err(e)) => {
self.cancel();
Err(Error::QueryError(e.to_string()))
}
None => {
self.finished = true;
Ok(None)
}
}
}
pub fn cancel(&mut self) {
if self.inner.is_null() {
return;
}
if !self.finished {
unsafe {
bindings::chdb_stream_cancel_query(self.conn_handle(), self.inner);
}
}
unsafe {
bindings::chdb_destroy_query_result(self.inner);
}
self.inner = std::ptr::null_mut();
self.finished = true;
}
}
fn is_eos_without_schema(ffi_stream: &mut FFI_ArrowArrayStream) -> Result<bool> {
let Some(get_schema) = ffi_stream.get_schema else {
return Err(Error::InvalidData(
"Arrow stream has no get_schema callback".into(),
));
};
let mut schema = FFI_ArrowSchema::empty();
let ret_code = unsafe { get_schema(ffi_stream, &mut schema) };
if ret_code == 0 {
return Ok(false);
}
let Some(get_next) = ffi_stream.get_next else {
return Err(Error::InvalidData(
"Arrow stream has no get_next callback".into(),
));
};
let mut array = FFI_ArrowArray::empty();
let next_code = unsafe { get_next(ffi_stream, &mut array) };
let detail = ffi_stream.get_last_error.and_then(|get_error| {
let ptr = unsafe { get_error(ffi_stream) };
(!ptr.is_null()).then(|| {
unsafe { CStr::from_ptr(ptr) }
.to_string_lossy()
.into_owned()
})
});
if next_code == 0 && array.release.is_none() && detail.is_none() {
return Ok(true);
}
Err(Error::QueryError(detail.unwrap_or_else(|| {
format!("Arrow stream schema failed ({ret_code}); next batch returned {next_code}")
})))
}
impl Iterator for ArrowQueryStream<'_> {
type Item = Result<RecordBatch>;
fn next(&mut self) -> Option<Self::Item> {
match self.next_batch() {
Ok(Some(batch)) => Some(Ok(batch)),
Ok(None) => None,
Err(err) => Some(Err(err)),
}
}
}
impl Drop for ArrowQueryStream<'_> {
fn drop(&mut self) {
if !self.inner.is_null() {
self.cancel();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::session::SessionBuilder;
use crate::test_utils::tempdir;
unsafe extern "C" fn failing_schema(
_: *mut FFI_ArrowArrayStream,
_: *mut FFI_ArrowSchema,
) -> i32 {
1
}
unsafe extern "C" fn empty_next(_: *mut FFI_ArrowArrayStream, _: *mut FFI_ArrowArray) -> i32 {
0
}
unsafe extern "C" fn schema_error(_: *mut FFI_ArrowArrayStream) -> *const c_char {
c"invalid schema".as_ptr()
}
#[test]
fn schema_error_is_not_silently_treated_as_eof() {
let mut stream = FFI_ArrowArrayStream::empty();
stream.get_schema = Some(failing_schema);
stream.get_next = Some(empty_next);
stream.get_last_error = Some(schema_error);
let err = is_eos_without_schema(&mut stream).unwrap_err();
assert!(err.to_string().contains("invalid schema"));
stream.get_last_error = None;
assert!(is_eos_without_schema(&mut stream).unwrap());
}
#[test]
fn test_arrow_query_stream_row_count_and_chunking() -> Result<()> {
let mut conn = Connection::open_in_memory()?;
let mut stream = conn.query_stream_arrow("SELECT number FROM numbers(100_000)")?;
let mut batches = 0usize;
let mut rows = 0usize;
while let Some(batch) = stream.next_batch()? {
batches += 1;
rows += batch.num_rows();
}
assert_eq!(rows, 100_000);
assert!(batches > 1, "expected multiple batches, got {batches}");
Ok(())
}
#[test]
fn test_execute_stream_arrow() -> Result<()> {
use crate::execute_stream_arrow;
let mut stream = execute_stream_arrow("SELECT number FROM numbers(10)")?;
let mut rows = 0usize;
while let Some(batch) = stream.next_batch()? {
rows += batch.num_rows();
}
assert_eq!(rows, 10);
Ok(())
}
#[test]
fn test_session_execute_stream_arrow() -> Result<()> {
let tmp = tempdir();
let mut session = SessionBuilder::new()
.with_data_path(tmp.path())
.with_auto_cleanup(true)
.build()?;
session.execute(
"CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
None,
)?;
session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
let mut stream = session.execute_stream_arrow("SELECT * FROM items")?;
let mut rows = 0usize;
while let Some(batch) = stream.next_batch()? {
rows += batch.num_rows();
}
assert_eq!(rows, 3);
Ok(())
}
#[test]
fn test_arrow_query_stream_iterator() -> Result<()> {
let mut conn = Connection::open_in_memory()?;
let stream = conn.query_stream_arrow("SELECT number FROM numbers(5)")?;
let batches: Vec<_> = stream.collect::<Result<Vec<_>>>()?;
assert_eq!(batches.len(), 1);
assert_eq!(batches[0].num_rows(), 5);
Ok(())
}
#[test]
fn test_arrow_query_stream_empty_filter_returns_none() -> Result<()> {
let tmp = tempdir();
let mut session = SessionBuilder::new()
.with_data_path(tmp.path())
.with_auto_cleanup(true)
.build()?;
session.execute(
"CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
None,
)?;
session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
let mut stream = session.execute_stream_arrow("SELECT * FROM items WHERE id > 100")?;
assert!(stream.next_batch()?.is_none());
assert!(stream.next_batch()?.is_none());
Ok(())
}
#[test]
fn test_arrow_query_stream_error_then_retry_returns_none() -> Result<()> {
let mut conn = Connection::open_in_memory()?;
let mut stream = conn.query_stream_arrow("SELECT * FROM nonexistent_table")?;
assert!(stream.next_batch().is_err());
assert!(stream.next_batch()?.is_none());
Ok(())
}
#[test]
fn test_arrow_query_stream_syntax_error_fails_at_start() -> Result<()> {
let mut conn = Connection::open_in_memory()?;
let result = conn.query_stream_arrow("SELECT invalid syntax here");
assert!(result.is_err());
Ok(())
}
#[test]
fn test_arrow_query_stream_with_params_empty_filter_returns_none() -> Result<()> {
let tmp = tempdir();
let mut session = SessionBuilder::new()
.with_data_path(tmp.path())
.with_auto_cleanup(true)
.build()?;
session.execute(
"CREATE TABLE items (id UInt64) ENGINE = MergeTree() ORDER BY id",
None,
)?;
session.execute("INSERT INTO items VALUES (1), (2), (3)", None)?;
let mut stream = session.execute_stream_arrow_with_params(
"SELECT * FROM items WHERE id > {min_id:UInt64}",
[("min_id", 100_u64)],
None,
)?;
assert!(stream.next_batch()?.is_none());
assert!(stream.next_batch()?.is_none());
Ok(())
}
#[test]
fn test_arrow_query_stream_with_params_error_then_retry_returns_none() -> Result<()> {
let mut conn = Connection::open_in_memory()?;
let mut stream = conn.query_stream_arrow_with_params(
"SELECT * FROM nonexistent_table WHERE id = {id:UInt64}",
[("id", 1_u64)],
None,
)?;
assert!(stream.next_batch().is_err());
assert!(stream.next_batch()?.is_none());
Ok(())
}
#[test]
fn test_arrow_query_stream_with_params_syntax_error_fails_at_start() -> Result<()> {
let mut conn = Connection::open_in_memory()?;
let result =
conn.query_stream_arrow_with_params("SELECT invalid syntax here", [("x", 1_u64)], None);
assert!(result.is_err());
Ok(())
}
#[test]
fn test_arrow_query_stream_early_drop() -> Result<()> {
let mut conn = Connection::open_in_memory()?;
let mut stream = conn.query_stream_arrow("SELECT number FROM numbers(1_000_000)")?;
let first = stream.next_batch()?.expect("expected a batch");
assert!(first.num_rows() > 0);
drop(stream);
Ok(())
}
}