1use arrow_schema::DataType;
46use std::ffi::CStr;
47use std::ptr::addr_of;
48use std::{
49 ffi::CString,
50 os::raw::{c_char, c_int, c_void},
51 sync::Arc,
52};
53
54use arrow_data::ffi::FFI_ArrowArray;
55use arrow_schema::{ArrowError, Field, FieldRef, Schema, SchemaRef, ffi::FFI_ArrowSchema};
56
57use crate::RecordBatchOptions;
58use crate::array::Array;
59use crate::array::StructArray;
60use crate::array::{ArrayRef, make_array};
61use crate::array_reader::ArrayReader;
62use crate::ffi::from_ffi_and_data_type;
63use crate::record_batch::{RecordBatch, RecordBatchReader};
64
65type Result<T> = std::result::Result<T, ArrowError>;
66
67#[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
70use libc::{EINVAL, EIO, ENOMEM, ENOSYS};
71
72#[cfg(all(target_family = "wasm", target_os = "unknown"))]
75const ENOMEM: i32 = 12;
76#[cfg(all(target_family = "wasm", target_os = "unknown"))]
77const EIO: i32 = 5;
78#[cfg(all(target_family = "wasm", target_os = "unknown"))]
79const EINVAL: i32 = 22;
80#[cfg(all(target_family = "wasm", target_os = "unknown"))]
81const ENOSYS: i32 = 38;
82
83#[repr(C)]
87#[derive(Debug)]
88pub struct FFI_ArrowArrayStream {
89 get_schema: Option<unsafe extern "C" fn(arg1: *mut Self, out: *mut FFI_ArrowSchema) -> c_int>,
93 get_next: Option<unsafe extern "C" fn(arg1: *mut Self, out: *mut FFI_ArrowArray) -> c_int>,
95 get_last_error: Option<unsafe extern "C" fn(arg1: *mut Self) -> *const c_char>,
97 release: Option<unsafe extern "C" fn(arg1: *mut Self)>,
99 private_data: *mut c_void,
101}
102
103unsafe impl Send for FFI_ArrowArrayStream {}
104
105unsafe extern "C" fn release_stream(stream: *mut FFI_ArrowArrayStream) {
107 if stream.is_null() {
108 return;
109 }
110 let stream = unsafe { &mut *stream };
111
112 stream.get_schema = None;
113 stream.get_next = None;
114 stream.get_last_error = None;
115
116 let private_data = unsafe { Box::from_raw(stream.private_data.cast::<StreamPrivateData>()) };
117 drop(private_data);
118
119 stream.release = None;
120}
121
122struct StreamPrivateData {
123 batch_reader: Box<dyn RecordBatchReader + Send>,
124 last_error: Option<CString>,
125}
126
127unsafe extern "C" fn get_schema(
129 stream: *mut FFI_ArrowArrayStream,
130 schema: *mut FFI_ArrowSchema,
131) -> c_int {
132 ExportedArrayStream { stream }.get_schema(schema)
133}
134
135unsafe extern "C" fn get_next(
137 stream: *mut FFI_ArrowArrayStream,
138 array: *mut FFI_ArrowArray,
139) -> c_int {
140 ExportedArrayStream { stream }.get_next(array)
141}
142
143unsafe extern "C" fn get_last_error(stream: *mut FFI_ArrowArrayStream) -> *const c_char {
145 let mut ffi_stream = ExportedArrayStream { stream };
146 match ffi_stream.get_last_error() {
149 Some(err_string) => err_string.as_ptr(),
150 None => std::ptr::null(),
151 }
152}
153
154impl Drop for FFI_ArrowArrayStream {
155 fn drop(&mut self) {
156 match self.release {
157 None => (),
158 Some(release) => unsafe { release(self) },
159 }
160 }
161}
162
163impl FFI_ArrowArrayStream {
164 pub fn new(batch_reader: Box<dyn RecordBatchReader + Send>) -> Self {
166 let private_data = Box::new(StreamPrivateData {
167 batch_reader,
168 last_error: None,
169 });
170
171 Self {
172 get_schema: Some(get_schema),
173 get_next: Some(get_next),
174 get_last_error: Some(get_last_error),
175 release: Some(release_stream),
176 private_data: Box::into_raw(private_data).cast::<c_void>(),
177 }
178 }
179
180 pub unsafe fn new_unchecked(
199 get_schema: Option<
200 unsafe extern "C" fn(arg1: *mut Self, out: *mut FFI_ArrowSchema) -> c_int,
201 >,
202 get_next: Option<unsafe extern "C" fn(arg1: *mut Self, out: *mut FFI_ArrowArray) -> c_int>,
203 get_last_error: Option<unsafe extern "C" fn(arg1: *mut Self) -> *const c_char>,
204 release: Option<unsafe extern "C" fn(arg1: *mut Self)>,
205 private_data: *mut c_void,
206 ) -> Self {
207 Self {
208 get_schema,
209 get_next,
210 get_last_error,
211 release,
212 private_data,
213 }
214 }
215
216 pub unsafe fn from_raw(raw_stream: *mut FFI_ArrowArrayStream) -> Self {
229 unsafe { std::ptr::replace(raw_stream, Self::empty()) }
230 }
231
232 pub fn empty() -> Self {
234 Self {
235 get_schema: None,
236 get_next: None,
237 get_last_error: None,
238 release: None,
239 private_data: std::ptr::null_mut(),
240 }
241 }
242
243 #[expect(clippy::unnecessary_safety_doc)]
244 pub fn get_schema(
250 &self,
251 ) -> Option<unsafe extern "C" fn(arg1: *mut Self, out: *mut FFI_ArrowSchema) -> c_int> {
252 self.get_schema
253 }
254
255 #[expect(clippy::unnecessary_safety_doc)]
256 pub fn get_next(
262 &self,
263 ) -> Option<unsafe extern "C" fn(arg1: *mut Self, out: *mut FFI_ArrowArray) -> c_int> {
264 self.get_next
265 }
266
267 #[expect(clippy::unnecessary_safety_doc)]
268 pub fn get_last_error(&self) -> Option<unsafe extern "C" fn(arg1: *mut Self) -> *const c_char> {
276 self.get_last_error
277 }
278
279 pub fn release(&self) -> Option<unsafe extern "C" fn(arg1: *mut Self)> {
281 self.release
282 }
283
284 pub fn private_data(&self) -> *mut c_void {
286 self.private_data
287 }
288
289 pub unsafe fn set_release(
301 &mut self,
302 release: Option<unsafe extern "C" fn(arg1: *mut Self)>,
303 ) -> Option<unsafe extern "C" fn(arg1: *mut Self)> {
304 std::mem::replace(&mut self.release, release)
305 }
306
307 pub unsafe fn set_private_data(&mut self, private_data: *mut c_void) -> *mut c_void {
315 std::mem::replace(&mut self.private_data, private_data)
316 }
317}
318
319struct ExportedArrayStream {
320 stream: *mut FFI_ArrowArrayStream,
321}
322
323impl ExportedArrayStream {
324 fn get_private_data(&mut self) -> &mut StreamPrivateData {
325 unsafe { &mut *(*self.stream).private_data.cast::<StreamPrivateData>() }
326 }
327
328 pub fn get_schema(&mut self, out: *mut FFI_ArrowSchema) -> i32 {
329 let private_data = self.get_private_data();
330 let reader = &private_data.batch_reader;
331
332 let schema = FFI_ArrowSchema::try_from(reader.schema().as_ref());
333
334 match schema {
335 Ok(schema) => {
336 unsafe { std::ptr::copy(addr_of!(schema), out, 1) };
337 std::mem::forget(schema);
338 0
339 }
340 Err(ref err) => {
341 private_data.last_error = Some(
342 CString::new(err.to_string()).expect("Error string has a null byte in it."),
343 );
344 get_error_code(err)
345 }
346 }
347 }
348
349 pub fn get_next(&mut self, out: *mut FFI_ArrowArray) -> i32 {
350 let private_data = self.get_private_data();
351 let reader = &mut private_data.batch_reader;
352
353 match reader.next() {
354 None => {
355 unsafe { std::ptr::write(out, FFI_ArrowArray::empty()) }
357 0
358 }
359 Some(next_batch) => {
360 if let Ok(batch) = next_batch {
361 let struct_array = StructArray::from(batch);
362 let array = FFI_ArrowArray::new(&struct_array.to_data());
363
364 unsafe { std::ptr::write_unaligned(out, array) };
365 0
366 } else {
367 let err = &next_batch.unwrap_err();
368 private_data.last_error = Some(
369 CString::new(err.to_string()).expect("Error string has a null byte in it."),
370 );
371 get_error_code(err)
372 }
373 }
374 }
375 }
376
377 pub fn get_last_error(&mut self) -> Option<&CString> {
378 self.get_private_data().last_error.as_ref()
379 }
380}
381
382fn get_error_code(err: &ArrowError) -> i32 {
383 match err {
384 ArrowError::NotYetImplemented(_) => ENOSYS,
385 ArrowError::MemoryError(_) => ENOMEM,
386 ArrowError::IoError(_, _) => EIO,
387 _ => EINVAL,
388 }
389}
390
391#[derive(Debug)]
399pub struct RecordBatchStreamReader {
400 stream: FFI_ArrowArrayStream,
401 schema: SchemaRef,
402}
403
404#[deprecated(since = "60.1.0", note = "Use `RecordBatchStreamReader` instead")]
406pub type ArrowArrayStreamReader = RecordBatchStreamReader;
407
408unsafe fn producer_error(stream_ptr: *mut FFI_ArrowArrayStream) -> Option<String> {
420 let get_last_error = unsafe { (*stream_ptr).get_last_error }?;
421
422 let error_str = unsafe { get_last_error(stream_ptr) };
423 if error_str.is_null() {
424 return None;
425 }
426
427 Some(
428 unsafe { CStr::from_ptr(error_str) }
429 .to_string_lossy()
430 .into_owned(),
431 )
432}
433
434fn get_stream_schema(stream_ptr: *mut FFI_ArrowArrayStream) -> Result<SchemaRef> {
437 let field = get_stream_field(stream_ptr)?;
438 match field.data_type() {
439 DataType::Struct(fields) => Ok(Arc::new(
440 Schema::new(fields.clone()).with_metadata(field.metadata().clone()),
441 )),
442 _ => Err(ArrowError::CDataInterface(
443 "Unable to interpret C data struct as a Schema".to_string(),
444 )),
445 }
446}
447
448fn get_stream_field(stream_ptr: *mut FFI_ArrowArrayStream) -> Result<FieldRef> {
451 let mut schema = FFI_ArrowSchema::empty();
452
453 let Some(get_schema) = (unsafe { (*stream_ptr).get_schema }) else {
454 return Err(ArrowError::CDataInterface(
455 "input stream does not provide the required get_schema callback".to_string(),
456 ));
457 };
458
459 let ret_code = unsafe { get_schema(stream_ptr, &raw mut schema) };
460
461 if ret_code == 0 {
462 Ok(Arc::new(Field::try_from(&schema)?))
463 } else {
464 let message = format!("Cannot get schema from input stream. Error code: {ret_code}");
465 let message = match unsafe { producer_error(stream_ptr) } {
468 Some(producer_message) => format!("{message}. Producer error: {producer_message}"),
469 None => message,
470 };
471 Err(ArrowError::CDataInterface(message))
472 }
473}
474
475impl RecordBatchStreamReader {
476 pub fn try_new(mut stream: FFI_ArrowArrayStream) -> Result<Self> {
479 if stream.release.is_none() {
480 return Err(ArrowError::CDataInterface(
481 "input stream is already released".to_string(),
482 ));
483 }
484
485 let schema = get_stream_schema(&raw mut stream)?;
486
487 Ok(Self { stream, schema })
488 }
489
490 pub unsafe fn from_raw(raw_stream: *mut FFI_ArrowArrayStream) -> Result<Self> {
501 Self::try_new(unsafe { FFI_ArrowArrayStream::from_raw(raw_stream) })
502 }
503}
504
505impl Iterator for RecordBatchStreamReader {
506 type Item = Result<RecordBatch>;
507
508 fn next(&mut self) -> Option<Self::Item> {
509 let mut array = FFI_ArrowArray::empty();
510
511 let Some(get_next) = self.stream.get_next else {
512 return Some(Err(ArrowError::CDataInterface(
513 "input stream does not provide the required get_next callback".to_string(),
514 )));
515 };
516
517 let ret_code = unsafe { get_next(&raw mut self.stream, &raw mut array) };
518
519 if ret_code == 0 {
520 if array.is_released() {
522 return None;
523 }
524
525 let result = unsafe {
526 from_ffi_and_data_type(array, DataType::Struct(self.schema().fields().clone()))
527 };
528 Some(result.and_then(|data| {
529 let len = data.len();
530 RecordBatch::try_new_with_options(
531 self.schema.clone(),
532 StructArray::from(data).into_parts().1,
533 &RecordBatchOptions::new().with_row_count(Some(len)),
534 )
535 }))
536 } else {
537 let message =
538 format!("Cannot get next batch from input stream. Error code: {ret_code}");
539 let message = match unsafe { producer_error(&raw mut self.stream) } {
542 Some(producer_message) => format!("{message}. Producer error: {producer_message}"),
543 None => message,
544 };
545 Some(Err(ArrowError::CDataInterface(message)))
546 }
547 }
548}
549
550impl RecordBatchReader for RecordBatchStreamReader {
551 fn schema(&self) -> SchemaRef {
552 self.schema.clone()
553 }
554}
555
556#[derive(Debug)]
562pub struct ArrayStreamReader {
563 stream: FFI_ArrowArrayStream,
564 field: FieldRef,
565}
566
567impl ArrayStreamReader {
568 pub fn try_new(mut stream: FFI_ArrowArrayStream) -> Result<Self> {
571 if stream.release.is_none() {
572 return Err(ArrowError::CDataInterface(
573 "input stream is already released".to_string(),
574 ));
575 }
576
577 let field = get_stream_field(&raw mut stream)?;
578
579 Ok(Self { stream, field })
580 }
581
582 pub unsafe fn from_raw(raw_stream: *mut FFI_ArrowArrayStream) -> Result<Self> {
588 Self::try_new(unsafe { FFI_ArrowArrayStream::from_raw(raw_stream) })
589 }
590}
591
592impl Iterator for ArrayStreamReader {
593 type Item = Result<ArrayRef>;
594
595 fn next(&mut self) -> Option<Self::Item> {
596 let mut array = FFI_ArrowArray::empty();
597
598 let ret_code =
599 unsafe { self.stream.get_next.unwrap()(&raw mut self.stream, &raw mut array) };
600
601 if ret_code == 0 {
602 if array.is_released() {
604 return None;
605 }
606
607 let result = unsafe { from_ffi_and_data_type(array, self.field.data_type().clone()) };
608 Some(result.map(make_array))
609 } else {
610 let message =
611 format!("Cannot get next array from input stream. Error code: {ret_code}");
612 let message = match unsafe { producer_error(&raw mut self.stream) } {
615 Some(producer_message) => format!("{message}. Producer error: {producer_message}"),
616 None => message,
617 };
618 Some(Err(ArrowError::CDataInterface(message)))
619 }
620 }
621}
622
623impl ArrayReader for ArrayStreamReader {
624 fn field(&self) -> &FieldRef {
625 &self.field
626 }
627}
628
629#[cfg(test)]
630mod tests {
631 use super::*;
632 use std::collections::HashMap;
633
634 use arrow_schema::Field;
635
636 use crate::array::Int32Array;
637 use crate::ffi::from_ffi;
638
639 struct TestRecordBatchReader {
640 schema: SchemaRef,
641 iter: Box<dyn Iterator<Item = Result<RecordBatch>> + Send>,
642 }
643
644 impl TestRecordBatchReader {
645 pub fn new(
646 schema: SchemaRef,
647 iter: Box<dyn Iterator<Item = Result<RecordBatch>> + Send>,
648 ) -> TestRecordBatchReader {
649 TestRecordBatchReader { schema, iter }
650 }
651 }
652
653 impl Iterator for TestRecordBatchReader {
654 type Item = Result<RecordBatch>;
655
656 fn next(&mut self) -> Option<Self::Item> {
657 self.iter.next()
658 }
659 }
660
661 impl RecordBatchReader for TestRecordBatchReader {
662 fn schema(&self) -> SchemaRef {
663 self.schema.clone()
664 }
665 }
666
667 fn _test_round_trip_export(batch: RecordBatch, schema: Arc<Schema>) -> Result<()> {
668 let iter = Box::new(vec![batch.clone(), batch.clone()].into_iter().map(Ok)) as _;
669
670 let reader = Box::new(TestRecordBatchReader::new(schema.clone(), iter));
671
672 let mut ffi_stream = FFI_ArrowArrayStream::new(reader);
674
675 let mut ffi_schema = FFI_ArrowSchema::empty();
677 let ret_code = unsafe { get_schema(&raw mut ffi_stream, &raw mut ffi_schema) };
678 assert_eq!(ret_code, 0);
679
680 let exported_schema = Schema::try_from(&ffi_schema).unwrap();
681 assert_eq!(&exported_schema, schema.as_ref());
682
683 let mut produced_batches = vec![];
685 loop {
686 let mut ffi_array = FFI_ArrowArray::empty();
687 let ret_code = unsafe { get_next(&raw mut ffi_stream, &raw mut ffi_array) };
688 assert_eq!(ret_code, 0);
689
690 if ffi_array.is_released() {
692 break;
693 }
694
695 let array = unsafe { from_ffi(ffi_array, &ffi_schema) }.unwrap();
696 let len = array.len();
697
698 let record_batch = RecordBatch::try_new_with_options(
699 SchemaRef::from(exported_schema.clone()),
700 StructArray::from(array).into_parts().1,
701 &RecordBatchOptions::new().with_row_count(Some(len)),
702 )
703 .unwrap();
704 produced_batches.push(record_batch);
705 }
706
707 assert_eq!(produced_batches, vec![batch.clone(), batch]);
708
709 Ok(())
710 }
711
712 fn _test_round_trip_import(batch: RecordBatch, schema: Arc<Schema>) -> Result<()> {
713 let iter = Box::new(vec![batch.clone(), batch.clone()].into_iter().map(Ok)) as _;
714
715 let reader = Box::new(TestRecordBatchReader::new(schema.clone(), iter));
716
717 let stream = FFI_ArrowArrayStream::new(reader);
719 let stream_reader = RecordBatchStreamReader::try_new(stream).unwrap();
720
721 let imported_schema = stream_reader.schema();
722 assert_eq!(imported_schema, schema);
723
724 let mut produced_batches = vec![];
725 for batch in stream_reader {
726 produced_batches.push(batch.unwrap());
727 }
728
729 assert_eq!(produced_batches, vec![batch.clone(), batch]);
730
731 Ok(())
732 }
733
734 #[test]
735 fn test_stream_round_trip() {
736 let array = Int32Array::from(vec![Some(2), None, Some(1), None]);
737 let array: Arc<dyn Array> = Arc::new(array);
738 let metadata = HashMap::from([("foo".to_owned(), "bar".to_owned())]);
739
740 let schema = Arc::new(Schema::new_with_metadata(
741 vec![
742 Field::new("a", array.data_type().clone(), true).with_metadata(metadata.clone()),
743 Field::new("b", array.data_type().clone(), true).with_metadata(metadata.clone()),
744 Field::new("c", array.data_type().clone(), true).with_metadata(metadata.clone()),
745 ],
746 metadata,
747 ));
748 let batch = RecordBatch::try_new(schema.clone(), vec![array.clone(), array.clone(), array])
749 .unwrap();
750
751 _test_round_trip_export(batch.clone(), schema.clone()).unwrap();
752 _test_round_trip_import(batch, schema).unwrap();
753 }
754
755 #[test]
756 fn test_stream_round_trip_no_columns() {
757 let metadata = HashMap::from([("foo".to_owned(), "bar".to_owned())]);
758
759 let schema = Arc::new(Schema::new_with_metadata(Vec::<Field>::new(), metadata));
760 let batch = RecordBatch::try_new_with_options(
761 schema.clone(),
762 Vec::<Arc<dyn Array>>::new(),
763 &RecordBatchOptions::new().with_row_count(Some(10)),
764 )
765 .unwrap();
766
767 _test_round_trip_export(batch.clone(), schema.clone()).unwrap();
768 _test_round_trip_import(batch, schema).unwrap();
769 }
770
771 #[test]
772 fn test_error_import() -> Result<()> {
773 let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
774
775 let iter =
776 Box::new(vec![Err(ArrowError::MemoryError("out of memory".to_string()))].into_iter());
777
778 let reader = Box::new(TestRecordBatchReader::new(schema.clone(), iter));
779
780 let stream = FFI_ArrowArrayStream::new(reader);
782 let stream_reader = RecordBatchStreamReader::try_new(stream).unwrap();
783
784 let imported_schema = stream_reader.schema();
785 assert_eq!(imported_schema, schema);
786
787 let mut produced_batches = vec![];
788 for batch in stream_reader {
789 produced_batches.push(batch);
790 }
791
792 assert_eq!(produced_batches.len(), 1);
794 assert_eq!(
795 produced_batches[0].as_ref().unwrap_err().to_string(),
796 format!(
797 "C Data interface error: Cannot get next batch from input stream. \
798 Error code: {ENOMEM}. Producer error: Memory error: out of memory"
799 )
800 );
801
802 Ok(())
803 }
804
805 unsafe extern "C" fn failing_get_schema(
806 _stream: *mut FFI_ArrowArrayStream,
807 _out: *mut FFI_ArrowSchema,
808 ) -> c_int {
809 EIO
810 }
811
812 unsafe extern "C" fn working_get_schema(
813 _stream: *mut FFI_ArrowArrayStream,
814 out: *mut FFI_ArrowSchema,
815 ) -> c_int {
816 let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
817 unsafe { std::ptr::write(out, FFI_ArrowSchema::try_from(&schema).unwrap()) };
818 0
819 }
820
821 unsafe extern "C" fn failing_get_next(
822 _stream: *mut FFI_ArrowArrayStream,
823 _out: *mut FFI_ArrowArray,
824 ) -> c_int {
825 EIO
826 }
827
828 unsafe extern "C" fn producer_last_error(_stream: *mut FFI_ArrowArrayStream) -> *const c_char {
829 c"the producer failed".as_ptr()
830 }
831
832 unsafe extern "C" fn null_last_error(_stream: *mut FFI_ArrowArrayStream) -> *const c_char {
833 std::ptr::null()
834 }
835
836 unsafe extern "C" fn mark_released(stream: *mut FFI_ArrowArrayStream) {
837 unsafe { (*stream).release = None };
838 }
839
840 fn failing_stream(
841 get_last_error: Option<unsafe extern "C" fn(*mut FFI_ArrowArrayStream) -> *const c_char>,
842 ) -> FFI_ArrowArrayStream {
843 let mut stream = FFI_ArrowArrayStream::empty();
844 stream.get_schema = Some(failing_get_schema);
845 stream.get_next = Some(failing_get_next);
846 stream.get_last_error = get_last_error;
847 stream.release = Some(mark_released);
848 stream
849 }
850
851 #[test]
852 fn test_import_stream_without_get_schema_callback() {
853 let stream = unsafe {
855 FFI_ArrowArrayStream::new_unchecked(
856 None,
857 Some(failing_get_next),
858 None,
859 Some(mark_released),
860 std::ptr::null_mut(),
861 )
862 };
863
864 let err = RecordBatchStreamReader::try_new(stream).unwrap_err();
865 assert!(matches!(err, ArrowError::CDataInterface(_)));
866 assert!(err.to_string().contains("get_schema"));
867 }
868
869 #[test]
870 fn test_import_stream_without_get_next_callback() {
871 let stream = unsafe {
873 FFI_ArrowArrayStream::new_unchecked(
874 Some(working_get_schema),
875 None,
876 None,
877 Some(mark_released),
878 std::ptr::null_mut(),
879 )
880 };
881
882 let err = RecordBatchStreamReader::try_new(stream)
883 .unwrap()
884 .next()
885 .unwrap()
886 .unwrap_err();
887 assert!(matches!(err, ArrowError::CDataInterface(_)));
888 assert!(err.to_string().contains("get_next"));
889 }
890
891 #[test]
892 fn test_import_schema_error_reports_producer_message() {
893 let err = RecordBatchStreamReader::try_new(failing_stream(Some(producer_last_error)))
894 .unwrap_err();
895 assert_eq!(
896 err.to_string(),
897 format!(
898 "C Data interface error: Cannot get schema from input stream. \
899 Error code: {EIO}. Producer error: the producer failed"
900 )
901 );
902 }
903
904 #[test]
905 fn test_import_schema_error_without_producer_message() {
906 let err =
909 RecordBatchStreamReader::try_new(failing_stream(Some(null_last_error))).unwrap_err();
910 assert_eq!(
911 err.to_string(),
912 format!(
913 "C Data interface error: Cannot get schema from input stream. Error code: {EIO}"
914 )
915 );
916 }
917
918 #[test]
919 fn test_import_schema_error_without_error_callback() {
920 let err = RecordBatchStreamReader::try_new(failing_stream(None)).unwrap_err();
921 assert_eq!(
922 err.to_string(),
923 format!(
924 "C Data interface error: Cannot get schema from input stream. Error code: {EIO}"
925 )
926 );
927 }
928
929 #[test]
930 fn test_import_next_error_without_producer_message() {
931 let mut stream = failing_stream(Some(null_last_error));
934 stream.get_schema = Some(working_get_schema);
935
936 let err = RecordBatchStreamReader::try_new(stream)
937 .unwrap()
938 .next()
939 .unwrap()
940 .unwrap_err();
941 assert_eq!(
942 err.to_string(),
943 format!(
944 "C Data interface error: Cannot get next batch from input stream. Error code: {EIO}"
945 )
946 );
947 }
948
949 static STREAM_WRAPPER_RAN: std::sync::atomic::AtomicBool =
953 std::sync::atomic::AtomicBool::new(false);
954
955 struct StreamWrapperData {
956 original_release: Option<unsafe extern "C" fn(*mut FFI_ArrowArrayStream)>,
957 original_private_data: *mut c_void,
958 }
959
960 unsafe extern "C" fn wrapping_release(stream: *mut FFI_ArrowArrayStream) {
961 use std::sync::atomic::Ordering;
962 let stream = unsafe { &mut *stream };
963 let data = unsafe { Box::from_raw(stream.private_data().cast::<StreamWrapperData>()) };
964 STREAM_WRAPPER_RAN.store(true, Ordering::SeqCst);
965 unsafe { stream.set_release(data.original_release) };
966 unsafe { stream.set_private_data(data.original_private_data) };
967 if let Some(release) = stream.release() {
968 unsafe { release(stream) };
969 }
970 }
971
972 #[test]
973 fn test_wrap_release_callback() {
974 use std::sync::atomic::Ordering;
975
976 let batch_reader = Box::new(TestRecordBatchReader::new(
977 Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])),
978 Box::new(std::iter::empty()),
979 ));
980 let mut stream = FFI_ArrowArrayStream::new(batch_reader);
981
982 let data = Box::new(StreamWrapperData {
983 original_release: stream.release(),
984 original_private_data: stream.private_data(),
985 });
986 unsafe { stream.set_release(Some(wrapping_release)) };
987 unsafe { stream.set_private_data(Box::into_raw(data).cast::<c_void>()) };
988
989 drop(stream); assert!(STREAM_WRAPPER_RAN.load(Ordering::SeqCst));
991 }
992
993 struct ArrayStreamPrivateData {
997 field: Field,
998 arrays: std::vec::IntoIter<Int32Array>,
999 }
1000
1001 unsafe extern "C" fn array_stream_get_schema(
1002 stream: *mut FFI_ArrowArrayStream,
1003 out: *mut FFI_ArrowSchema,
1004 ) -> c_int {
1005 let private_data = unsafe { &*(*stream).private_data().cast::<ArrayStreamPrivateData>() };
1006 let schema = FFI_ArrowSchema::try_from(&private_data.field).unwrap();
1007 unsafe { std::ptr::write(out, schema) };
1008 0
1009 }
1010
1011 unsafe extern "C" fn array_stream_get_next(
1012 stream: *mut FFI_ArrowArrayStream,
1013 out: *mut FFI_ArrowArray,
1014 ) -> c_int {
1015 let private_data =
1016 unsafe { &mut *(*stream).private_data().cast::<ArrayStreamPrivateData>() };
1017 match private_data.arrays.next() {
1018 None => unsafe { std::ptr::write(out, FFI_ArrowArray::empty()) },
1020 Some(array) => unsafe {
1021 std::ptr::write_unaligned(out, FFI_ArrowArray::new(&array.to_data()))
1022 },
1023 }
1024 0
1025 }
1026
1027 unsafe extern "C" fn array_stream_get_last_error(
1028 _stream: *mut FFI_ArrowArrayStream,
1029 ) -> *const c_char {
1030 std::ptr::null()
1031 }
1032
1033 unsafe extern "C" fn array_stream_release(stream: *mut FFI_ArrowArrayStream) {
1034 let private_data =
1035 unsafe { Box::from_raw((*stream).private_data().cast::<ArrayStreamPrivateData>()) };
1036 drop(private_data);
1037 unsafe { std::ptr::write(stream, FFI_ArrowArrayStream::empty()) };
1040 }
1041
1042 fn array_stream(field: Field, arrays: Vec<Int32Array>) -> FFI_ArrowArrayStream {
1043 let private_data = Box::new(ArrayStreamPrivateData {
1044 field,
1045 arrays: arrays.into_iter(),
1046 });
1047
1048 unsafe {
1049 FFI_ArrowArrayStream::new_unchecked(
1050 Some(array_stream_get_schema),
1051 Some(array_stream_get_next),
1052 Some(array_stream_get_last_error),
1053 Some(array_stream_release),
1054 Box::into_raw(private_data).cast::<c_void>(),
1055 )
1056 }
1057 }
1058
1059 #[test]
1060 fn test_new_unchecked_exports_stream_of_arrays() {
1061 let field = Field::new("a", DataType::Int32, true);
1062 let arrays = vec![
1063 Int32Array::from(vec![1, 2, 3]),
1064 Int32Array::from(vec![4, 5]),
1065 ];
1066 let mut stream = array_stream(field.clone(), arrays.clone());
1067
1068 let get_schema_fn = stream.get_schema().unwrap();
1071 let get_next_fn = stream.get_next().unwrap();
1072 let get_last_error_fn = stream.get_last_error().unwrap();
1073
1074 let mut ffi_schema = FFI_ArrowSchema::empty();
1075 let ret_code = unsafe { get_schema_fn(&raw mut stream, &raw mut ffi_schema) };
1076 assert_eq!(ret_code, 0);
1077 assert_eq!(Field::try_from(&ffi_schema).unwrap(), field);
1078
1079 let mut produced = vec![];
1080 loop {
1081 let mut ffi_array = FFI_ArrowArray::empty();
1082 let ret_code = unsafe { get_next_fn(&raw mut stream, &raw mut ffi_array) };
1083 assert_eq!(ret_code, 0);
1084
1085 if ffi_array.is_released() {
1087 break;
1088 }
1089
1090 let data = unsafe { from_ffi(ffi_array, &ffi_schema) }.unwrap();
1091 produced.push(Int32Array::from(data));
1092 }
1093
1094 assert_eq!(produced, arrays);
1095
1096 assert!(unsafe { get_last_error_fn(&raw mut stream) }.is_null());
1098
1099 drop(stream);
1101 }
1102
1103 #[test]
1104 fn test_accessors_report_an_empty_stream_as_released() {
1105 let stream = FFI_ArrowArrayStream::empty();
1106 assert!(stream.get_schema().is_none());
1107 assert!(stream.get_next().is_none());
1108 assert!(stream.get_last_error().is_none());
1109 assert!(stream.release().is_none());
1110 assert!(stream.private_data().is_null());
1111 }
1112 #[test]
1113 fn test_import_array_stream() {
1114 let field = Field::new("a", DataType::Int32, true);
1115 let arrays = vec![
1116 Int32Array::from(vec![1, 2, 3]),
1117 Int32Array::from(vec![4, 5]),
1118 ];
1119 let reader = ArrayStreamReader::try_new(array_stream(field, arrays.clone())).unwrap();
1120
1121 let produced = reader
1122 .map(|array| Int32Array::from(array.unwrap().to_data()))
1123 .collect::<Vec<_>>();
1124
1125 assert_eq!(produced, arrays);
1126 }
1127
1128 #[test]
1129 fn test_import_array_stream_reports_its_field() {
1130 let field = Field::new("a", DataType::Int32, true)
1131 .with_metadata(HashMap::from([("key".to_string(), "value".to_string())]));
1132 let reader = ArrayStreamReader::try_new(array_stream(field.clone(), vec![])).unwrap();
1133
1134 assert_eq!(reader.field().as_ref(), &field);
1135 }
1136
1137 #[test]
1138 fn test_import_array_stream_rejects_released_stream() {
1139 let err = ArrayStreamReader::try_new(FFI_ArrowArrayStream::empty()).unwrap_err();
1140
1141 assert_eq!(
1142 err.to_string(),
1143 "C Data interface error: input stream is already released"
1144 );
1145 }
1146
1147 #[test]
1148 fn test_import_array_schema_error_reports_producer_message() {
1149 let err =
1150 ArrayStreamReader::try_new(failing_stream(Some(producer_last_error))).unwrap_err();
1151
1152 assert_eq!(
1153 err.to_string(),
1154 format!(
1155 "C Data interface error: Cannot get schema from input stream. \
1156 Error code: {EIO}. Producer error: the producer failed"
1157 )
1158 );
1159 }
1160
1161 #[test]
1162 fn test_import_array_next_error_reports_producer_message() {
1163 let mut stream = failing_stream(Some(producer_last_error));
1164 stream.get_schema = Some(working_get_schema);
1165
1166 let err = ArrayStreamReader::try_new(stream)
1167 .unwrap()
1168 .next()
1169 .unwrap()
1170 .unwrap_err();
1171
1172 assert_eq!(
1173 err.to_string(),
1174 format!(
1175 "C Data interface error: Cannot get next array from input stream. \
1176 Error code: {EIO}. Producer error: the producer failed"
1177 )
1178 );
1179 }
1180 #[test]
1181 fn test_import_record_batch_stream_rejects_non_struct_stream() {
1182 let field = Field::new("a", DataType::Int32, true);
1185
1186 let err = RecordBatchStreamReader::try_new(array_stream(field, vec![])).unwrap_err();
1187
1188 assert_eq!(
1189 err.to_string(),
1190 "C Data interface error: Unable to interpret C data struct as a Schema"
1191 );
1192 }
1193}