use super::*;
use crate::export::arrow_convert_util::{
checked_binary_offsets, checked_offset, checked_string_offsets, checked_value_bytes,
};
use crate::export::arrow_schema::cql_type_to_arrow_data_type;
use crate::query::{ColumnInfo, QueryRow};
use crate::schema::CqlType;
use crate::types::{DataType, Value};
use crate::RowKey;
use arrow::array::{Array, Float32Array, Int32Array, StringArray};
use arrow::datatypes::DataType as ArrowDataType;
use std::collections::HashMap;
use std::sync::Arc;
fn col(name: &str, data_type: DataType, cql_type: Option<CqlType>) -> ColumnInfo {
ColumnInfo {
name: name.to_string(),
data_type,
nullable: true,
position: 0,
table_name: None,
cql_type,
}
}
fn row_one(name: &str, value: Value) -> QueryRow {
let mut values: HashMap<Arc<str>, Value> = HashMap::new();
values.insert(name.into(), value);
QueryRow {
values,
key: RowKey::new(Vec::new()),
metadata: Default::default(),
cell_metadata: None,
}
}
fn row_absent() -> QueryRow {
QueryRow {
values: HashMap::new(),
key: RowKey::new(Vec::new()),
metadata: Default::default(),
cell_metadata: None,
}
}
fn is_invalid_value(res: Result<arrow::record_batch::RecordBatch, ArrowConvertError>) -> bool {
matches!(res, Err(ArrowConvertError::InvalidValue(_)))
}
#[test]
fn typed_scalar_type_mismatch_is_error() {
let columns = vec![col("d", DataType::Timestamp, Some(CqlType::Date))];
let rows = vec![row_one("d", Value::Text("not-a-date".into()))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn flat_builder_type_mismatch_is_error() {
let columns = vec![col("n", DataType::Integer, None)];
let rows = vec![row_one("n", Value::Text("nope".into()))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn collection_expected_list_got_scalar_is_error() {
let columns = vec![col(
"l",
DataType::List,
Some(CqlType::List(Box::new(CqlType::Int))),
)];
let rows = vec![row_one("l", Value::Integer(5))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn collection_mistyped_element_is_error() {
let columns = vec![col(
"l",
DataType::List,
Some(CqlType::List(Box::new(CqlType::Int))),
)];
let rows = vec![row_one(
"l",
Value::List(vec![Value::Integer(1), Value::Text("bad".into())]),
)];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn collection_expected_map_got_scalar_is_error() {
let columns = vec![col(
"m",
DataType::Map,
Some(CqlType::Map(
Box::new(CqlType::Text),
Box::new(CqlType::Int),
)),
)];
let rows = vec![row_one("m", Value::Integer(7))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn null_and_absent_still_build_ok() {
let columns = vec![col("n", DataType::Integer, None)];
let rows = vec![row_one("n", Value::Null), row_absent()];
let batch = rows_to_record_batch(&columns, &rows).expect("null/absent must build");
assert_eq!(batch.num_rows(), 2);
assert_eq!(batch.column(0).null_count(), 2);
}
#[test]
fn correctly_typed_value_builds_ok() {
let columns = vec![col("n", DataType::Integer, None)];
let rows = vec![row_one("n", Value::Integer(42))];
let batch = rows_to_record_batch(&columns, &rows).expect("well-typed value must build");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.expect("Int32Array");
assert_eq!(arr.value(0), 42);
assert_eq!(arr.null_count(), 0);
}
#[test]
fn decimal_scale_above_fixed_is_error() {
let columns = vec![col("d", DataType::Blob, Some(CqlType::Decimal))];
let unscaled = num_bigint::BigInt::from(123_456_789_012i64).to_signed_bytes_be();
let rows = vec![row_one(
"d",
Value::Decimal {
scale: 12,
unscaled,
},
)];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn decimal_scale_within_fixed_builds_ok() {
use arrow::array::Decimal128Array;
let columns = vec![col("d", DataType::Blob, Some(CqlType::Decimal))];
let unscaled = num_bigint::BigInt::from(123_456i64).to_signed_bytes_be();
let rows = vec![row_one("d", Value::Decimal { scale: 3, unscaled })];
let batch = rows_to_record_batch(&columns, &rows).expect("in-range decimal must build");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<Decimal128Array>()
.expect("Decimal128Array");
assert_eq!(arr.value(0), 123_456_000_000i128);
assert_eq!(arr.null_count(), 0);
}
#[test]
fn decimal_null_and_absent_still_null() {
let columns = vec![col("d", DataType::Blob, Some(CqlType::Decimal))];
let rows = vec![row_one("d", Value::Null), row_absent()];
let batch = rows_to_record_batch(&columns, &rows).expect("null/absent decimal must build");
assert_eq!(batch.num_rows(), 2);
assert_eq!(batch.column(0).null_count(), 2);
}
#[test]
fn float32_column_accepts_wide_float_value() {
let flat = vec![col("h", DataType::Float32, None)];
let rows = vec![row_one("h", Value::Float(1.84f32 as f64))];
let batch = rows_to_record_batch(&flat, &rows).expect("wide float must narrow, not error");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<Float32Array>()
.expect("Float32Array");
assert_eq!(arr.value(0), 1.84f32);
assert_eq!(arr.null_count(), 0);
let typed = vec![col("h", DataType::Float32, Some(CqlType::Float))];
let rows = vec![row_one("h", Value::Float(1.84f32 as f64))];
let batch =
rows_to_record_batch(&typed, &rows).expect("wide float (typed) must narrow, not error");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<Float32Array>()
.expect("Float32Array");
assert_eq!(arr.value(0), 1.84f32);
}
#[test]
fn tuple_expected_tuple_got_scalar_is_error() {
let columns = vec![col(
"t",
DataType::Text,
Some(CqlType::Tuple(vec![CqlType::Int, CqlType::Text])),
)];
let rows = vec![row_one("t", Value::Text("not-a-tuple".into()))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn tuple_null_and_absent_still_build_ok() {
let columns = vec![col(
"t",
DataType::Text,
Some(CqlType::Tuple(vec![CqlType::Int, CqlType::Text])),
)];
let rows = vec![row_one("t", Value::Null), row_absent()];
let batch = rows_to_record_batch(&columns, &rows).expect("null/absent tuple must build");
assert_eq!(batch.num_rows(), 2);
assert_eq!(batch.column(0).null_count(), 2);
}
#[test]
fn udt_expected_udt_got_scalar_is_error() {
let columns = vec![col(
"u",
DataType::Text,
Some(CqlType::Udt(
"my_type".into(),
vec![("a".into(), CqlType::Int), ("b".into(), CqlType::Text)],
)),
)];
let rows = vec![row_one("u", Value::Integer(9))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn udt_null_and_absent_still_build_ok() {
let columns = vec![col(
"u",
DataType::Text,
Some(CqlType::Udt(
"my_type".into(),
vec![("a".into(), CqlType::Int), ("b".into(), CqlType::Text)],
)),
)];
let rows = vec![row_one("u", Value::Null), row_absent()];
let batch = rows_to_record_batch(&columns, &rows).expect("null/absent UDT must build");
assert_eq!(batch.num_rows(), 2);
assert_eq!(batch.column(0).null_count(), 2);
}
#[test]
fn empty_field_udt_expected_udt_got_scalar_is_error() {
let columns = vec![col(
"u",
DataType::Text,
Some(CqlType::Udt("unresolved".into(), vec![])),
)];
let rows = vec![row_one("u", Value::Integer(9))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn empty_field_tuple_expected_tuple_got_scalar_is_error() {
let columns = vec![col("t", DataType::Text, Some(CqlType::Tuple(vec![])))];
let rows = vec![row_one("t", Value::Text("nope".into()))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn authoritative_text_column_type_mismatch_is_error() {
for cql in [CqlType::Text, CqlType::Ascii, CqlType::Varchar] {
let columns = vec![col("s", DataType::Text, Some(cql))];
let rows = vec![row_one("s", Value::Integer(1))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
}
#[test]
fn authoritative_text_column_rejects_json() {
let columns = vec![col("s", DataType::Text, Some(CqlType::Text))];
let rows = vec![row_one(
"s",
Value::Json(Box::new(serde_json::json!({"a": 1}))),
)];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn frozen_wrapped_scalar_values_build_ok() {
let text_cols = vec![col(
"s",
DataType::Text,
Some(CqlType::Frozen(Box::new(CqlType::Text))),
)];
let text_rows = vec![row_one(
"s",
Value::Frozen(Box::new(Value::Text("hi".into()))),
)];
let batch =
rows_to_record_batch(&text_cols, &text_rows).expect("frozen<text> value must build");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.expect("StringArray");
assert_eq!(arr.value(0), "hi");
let date_cols = vec![col(
"d",
DataType::Integer,
Some(CqlType::Frozen(Box::new(CqlType::Date))),
)];
let date_rows = vec![row_one("d", Value::Frozen(Box::new(Value::Date(19_000))))];
let batch =
rows_to_record_batch(&date_cols, &date_rows).expect("frozen<date> value must build");
assert_eq!(batch.num_rows(), 1);
assert_eq!(batch.column(0).null_count(), 0);
}
#[test]
fn authoritative_text_column_builds_ok() {
let columns = vec![col("s", DataType::Text, Some(CqlType::Text))];
let rows = vec![
row_one("s", Value::Text("hi".into())),
row_one("s", Value::Null),
row_absent(),
];
let batch = rows_to_record_batch(&columns, &rows).expect("well-typed text must build");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.expect("StringArray");
assert_eq!(arr.value(0), "hi");
assert_eq!(arr.null_count(), 2);
}
#[test]
fn authoritative_int_column_rejects_date() {
let columns = vec![col("n", DataType::Integer, Some(CqlType::Int))];
let rows = vec![row_one("n", Value::Date(19_000))];
assert!(is_invalid_value(rows_to_record_batch(&columns, &rows)));
}
#[test]
fn opaque_int_column_accepts_date() {
let columns = vec![col("n", DataType::Integer, None)];
let rows = vec![row_one("n", Value::Date(19_000))];
let batch = rows_to_record_batch(&columns, &rows).expect("opaque int accepts Date");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.expect("Int32Array");
assert_eq!(arr.value(0), 19_000);
}
#[test]
fn authoritative_bigint_counter_reject_mismatch() {
let bigint_time = vec![col("b", DataType::BigInt, Some(CqlType::BigInt))];
assert!(is_invalid_value(rows_to_record_batch(
&bigint_time,
&[row_one("b", Value::Time(123))]
)));
let counter_time = vec![col("c", DataType::BigInt, Some(CqlType::Counter))];
assert!(is_invalid_value(rows_to_record_batch(
&counter_time,
&[row_one("c", Value::Time(123))]
)));
let bigint_counter = vec![col("b", DataType::BigInt, Some(CqlType::BigInt))];
assert!(is_invalid_value(rows_to_record_batch(
&bigint_counter,
&[row_one("b", Value::Counter(7))]
)));
}
#[test]
fn authoritative_counter_column_accepts_counter() {
let columns = vec![col("c", DataType::BigInt, Some(CqlType::Counter))];
let rows = vec![row_one("c", Value::Counter(42))];
let batch = rows_to_record_batch(&columns, &rows).expect("counter accepts Counter");
assert_eq!(batch.num_rows(), 1);
assert_eq!(batch.column(0).null_count(), 0);
}
#[test]
fn checked_offset_past_i32_max_is_error() {
assert_eq!(checked_offset(i32::MAX as usize).ok(), Some(i32::MAX));
assert!(matches!(
checked_offset(i32::MAX as usize + 1),
Err(ArrowConvertError::InvalidValue(_))
));
}
#[test]
fn checked_offset_normal_sizes_are_identity() {
assert_eq!(checked_offset(0).ok(), Some(0));
assert_eq!(checked_offset(1).ok(), Some(1));
assert_eq!(checked_offset(1_000_000).ok(), Some(1_000_000));
}
#[test]
fn checked_value_bytes_past_i32_max_is_error() {
assert!(checked_value_bytes(i32::MAX as usize).is_ok());
assert!(matches!(
checked_value_bytes(i32::MAX as usize + 1),
Err(ArrowConvertError::InvalidValue(_))
));
assert!(checked_value_bytes(0).is_ok());
assert!(checked_value_bytes(1_000_000).is_ok());
}
#[test]
fn typed_blob_builder_over_i32_max_fails_closed_without_2gib_clone() {
const CHUNK: usize = 16 * 1024 * 1024; const N: usize = 128; let big = Value::blob(vec![0u8; CHUNK]);
let refs: Vec<Option<&Value>> = (0..N).map(|_| Some(&big)).collect();
let err = super::build_typed_value_array(&CqlType::Blob, &refs);
assert!(
matches!(err, Err(ArrowConvertError::InvalidValue(_))),
"Blob arm must fail closed at the i32 offset ceiling"
);
}
#[test]
fn typed_text_builder_over_i32_max_fails_closed_without_2gib_clone() {
const CHUNK: usize = 16 * 1024 * 1024; const N: usize = 128; let big = Value::text("a".repeat(CHUNK));
let refs: Vec<Option<&Value>> = (0..N).map(|_| Some(&big)).collect();
let err = super::build_typed_value_array(&CqlType::Text, &refs);
assert!(
matches!(err, Err(ArrowConvertError::InvalidValue(_))),
"Text arm must fail closed at the i32 offset ceiling"
);
}
#[test]
fn opaque_text_fallback_over_i32_max_fails_closed_without_2gib_clone() {
use std::borrow::Cow;
const CHUNK: usize = 16 * 1024 * 1024; const N: usize = 128; let big = "a".repeat(CHUNK);
let refs: Vec<Option<Cow<str>>> = (0..N).map(|_| Some(Cow::Borrowed(big.as_str()))).collect();
let total: usize = refs.iter().flatten().map(|s| s.len()).sum();
assert_eq!(total, i32::MAX as usize + 1, "test must cross i32::MAX");
assert!(
matches!(
checked_string_offsets(&refs),
Err(ArrowConvertError::InvalidValue(_))
),
"opaque untyped Text fallback must fail closed at the i32 offset ceiling"
);
}
#[test]
fn opaque_text_fallback_preserves_raw_text_verbatim() {
use arrow::array::StringArray;
let cols = vec![col("o", DataType::Text, None)];
let rows = vec![
row_one("o", Value::Text("verbatim".into())),
row_one("o", Value::Null),
];
let batch = rows_to_record_batch(&cols, &rows).expect("opaque text must build");
let arr = batch
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.expect("Utf8 array");
assert_eq!(arr.value(0), "verbatim");
assert!(arr.is_null(1));
}
#[test]
fn scalar_binary_cumulative_bytes_over_i32_max_is_typed_error() {
const CHUNK: usize = 16 * 1024 * 1024; const N: usize = 128; let buf = vec![0u8; CHUNK];
let refs: Vec<Option<&[u8]>> = (0..N).map(|_| Some(buf.as_slice())).collect();
let total: usize = refs.iter().flatten().map(|b| b.len()).sum();
assert_eq!(total, i32::MAX as usize + 1, "test must cross i32::MAX");
assert!(matches!(
checked_binary_offsets(&refs),
Err(ArrowConvertError::InvalidValue(_))
));
}
#[test]
fn scalar_string_cumulative_bytes_over_i32_max_is_typed_error() {
let ok = vec![Some("a".to_string()), None, Some("bc".to_string())];
assert!(checked_string_offsets(&ok).is_ok());
assert!(matches!(
checked_value_bytes(i32::MAX as usize + 42),
Err(ArrowConvertError::InvalidValue(_))
));
}
#[test]
fn normal_scalar_text_and_blob_still_build_through_byte_guard() {
let text_cols = vec![col("t", DataType::Text, Some(CqlType::Text))];
let text_rows = vec![
row_one("t", Value::Text("hello".into())),
row_one("t", Value::Null),
];
let batch = rows_to_record_batch(&text_cols, &text_rows).expect("text must build");
assert_eq!(batch.num_rows(), 2);
let blob_cols = vec![col("b", DataType::Blob, Some(CqlType::Blob))];
let blob_rows = vec![row_one("b", Value::blob(vec![1, 2, 3, 4]))];
let batch = rows_to_record_batch(&blob_cols, &blob_rows).expect("blob must build");
assert_eq!(batch.num_rows(), 1);
}
#[test]
fn normal_collections_still_build_through_checked_offsets() {
let list_cols = vec![col(
"l",
DataType::List,
Some(CqlType::List(Box::new(CqlType::Int))),
)];
let list_rows = vec![
row_one("l", Value::List(vec![Value::Integer(1), Value::Integer(2)])),
row_one("l", Value::Null),
];
let batch = rows_to_record_batch(&list_cols, &list_rows).expect("list must build");
assert_eq!(batch.num_rows(), 2);
let map_cols = vec![col(
"m",
DataType::Map,
Some(CqlType::Map(
Box::new(CqlType::Text),
Box::new(CqlType::Int),
)),
)];
let map_rows = vec![row_one(
"m",
Value::Map(vec![(Value::Text("k".into()), Value::Integer(9))]),
)];
let batch = rows_to_record_batch(&map_cols, &map_rows).expect("map must build");
assert_eq!(batch.num_rows(), 1);
}
fn two_text_columns() -> (Vec<ColumnInfo>, Vec<QueryRow>) {
let columns = vec![
col("alpha", DataType::Text, Some(CqlType::Text)),
col("beta", DataType::Text, Some(CqlType::Text)),
];
let mut values: HashMap<Arc<str>, Value> = HashMap::new();
values.insert("alpha".into(), Value::Text("A".into()));
values.insert("beta".into(), Value::Text("B".into()));
let rows = vec![QueryRow {
values,
key: RowKey::new(Vec::new()),
metadata: Default::default(),
cell_metadata: None,
}];
(columns, rows)
}
#[test]
fn a_reordered_same_type_schema_is_rejected_not_silently_mislabeled() {
let (columns, rows) = two_text_columns();
let reordered: Vec<ColumnInfo> = columns.iter().rev().cloned().collect();
let reordered_schema = Arc::new(build_arrow_schema(&reordered).expect("schema"));
assert_eq!(
reordered_schema
.fields()
.iter()
.map(|f| f.name().as_str())
.collect::<Vec<_>>(),
vec!["beta", "alpha"],
"the fixture must actually be reordered"
);
let err = rows_to_record_batch_with_schema(Arc::clone(&reordered_schema), &columns, &rows)
.expect_err("a reordered schema must be rejected");
match &err {
ArrowConvertError::SchemaMismatch(msg) => {
assert!(
msg.contains("field 0 is 'beta'") && msg.contains("column 0 is 'alpha'"),
"the error must name the offending position and both names, got: {msg}"
);
}
other => panic!("expected SchemaMismatch, got {other:?}"),
}
let arrays = convert_to_arrays(&columns, &rows).expect("arrays");
let arrow_accepted = arrow::record_batch::RecordBatch::try_new(reordered_schema, arrays)
.expect("RecordBatch::try_new compares field TYPES and lengths only");
assert_eq!(
arrow_accepted.schema().field(0).name(),
"beta",
"Arrow labelled column 0 'beta'…"
);
let mislabeled = arrow_accepted
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.expect("utf8");
assert_eq!(
mislabeled.value(0),
"A",
"…while it holds ALPHA's value — exactly the silent mislabeling the \
rejection above prevents"
);
}
#[test]
fn a_schema_with_the_wrong_field_count_is_rejected() {
let (columns, rows) = two_text_columns();
let one_column_schema = Arc::new(build_arrow_schema(&columns[..1]).expect("schema"));
let err = rows_to_record_batch_with_schema(one_column_schema, &columns, &rows)
.expect_err("an arity mismatch must be rejected");
assert!(
matches!(&err, ArrowConvertError::SchemaMismatch(m)
if m.contains("1 field(s)") && m.contains("2 column(s)")),
"got {err:?}"
);
}
#[test]
fn the_matching_schema_path_is_unchanged() {
let (columns, rows) = two_text_columns();
let schema = Arc::new(build_arrow_schema(&columns).expect("schema"));
let with_schema = rows_to_record_batch_with_schema(schema, &columns, &rows)
.expect("the matching schema must be accepted");
let built_inline = rows_to_record_batch(&columns, &rows).expect("inline schema");
assert_eq!(with_schema.schema(), built_inline.schema());
assert_eq!(with_schema.num_rows(), built_inline.num_rows());
assert_eq!(
with_schema
.schema()
.fields()
.iter()
.map(|f| f.name().as_str())
.collect::<Vec<_>>(),
vec!["alpha", "beta"]
);
}
fn uuid_and_text_columns() -> (Vec<ColumnInfo>, Vec<QueryRow>) {
let columns = vec![
col("id", DataType::Uuid, Some(CqlType::Uuid)),
col("label", DataType::Text, Some(CqlType::Text)),
];
let mut values: HashMap<Arc<str>, Value> = HashMap::new();
values.insert("id".into(), Value::Uuid([7u8; 16]));
values.insert("label".into(), Value::Text("L".into()));
let rows = vec![QueryRow {
values,
key: RowKey::new(Vec::new()),
metadata: Default::default(),
cell_metadata: None,
}];
(columns, rows)
}
fn schema_with<F: FnMut(usize, Field) -> Field>(
columns: &[ColumnInfo],
mut mutate: F,
) -> Arc<Schema> {
let built = build_arrow_schema(columns).expect("schema");
let fields: Vec<Field> = built
.fields()
.iter()
.enumerate()
.map(|(i, f)| mutate(i, f.as_ref().clone()))
.collect();
Arc::new(Schema::new(fields))
}
fn expect_schema_mismatch(res: Result<RecordBatch, ArrowConvertError>) -> String {
match res {
Err(ArrowConvertError::SchemaMismatch(msg)) => msg,
Err(other) => panic!("expected SchemaMismatch, got {other:?}"),
Ok(batch) => panic!(
"expected SchemaMismatch, got a batch labelled {:?}",
batch.schema()
),
}
}
fn try_new_accepts(schema: Arc<Schema>, columns: &[ColumnInfo], rows: &[QueryRow]) -> RecordBatch {
let arrays = convert_to_arrays(columns, rows).expect("arrays");
RecordBatch::try_new(schema, arrays)
.expect("RecordBatch::try_new must ACCEPT this schema, or the test proves nothing")
}
#[test]
fn a_renamed_same_type_field_is_rejected_and_arrow_would_accept_it() {
let (columns, rows) = two_text_columns();
let renamed = schema_with(&columns, |i, f| {
if i == 0 {
Field::new("renamed", f.data_type().clone(), f.is_nullable())
} else {
f
}
});
let msg = expect_schema_mismatch(rows_to_record_batch_with_schema(
Arc::clone(&renamed),
&columns,
&rows,
));
assert!(
msg.contains("field 0 is 'renamed'") && msg.contains("column 0 is 'alpha'"),
"the message must name the position and both names, got: {msg}"
);
let accepted = try_new_accepts(renamed, &columns, &rows);
assert_eq!(
accepted.schema().field(0).name(),
"renamed",
"Arrow labelled alpha's values 'renamed' — the silent mislabeling the \
rejection prevents"
);
}
#[test]
fn a_nullability_flip_is_rejected_and_arrow_would_accept_it() {
let (columns, rows) = uuid_and_text_columns();
assert!(
columns.iter().all(|c| c.nullable),
"the fixture's columns must map to nullable fields for the flip to be a \
difference"
);
let flipped = schema_with(
&columns,
|i, f| if i == 1 { f.with_nullable(false) } else { f },
);
let msg = expect_schema_mismatch(rows_to_record_batch_with_schema(
Arc::clone(&flipped),
&columns,
&rows,
));
assert!(
msg.contains("field 1 'label'")
&& msg.contains("nullable=false")
&& msg.contains("nullable=true"),
"the message must name the position and both nullability values, got: {msg}"
);
let accepted = try_new_accepts(flipped, &columns, &rows);
assert!(
!accepted.schema().field(1).is_nullable(),
"Arrow accepted the batch and declared a nullable column NON-nullable — a \
schema every consumer of this batch would read as a guarantee"
);
assert_eq!(
accepted.column(1).null_count(),
0,
"the fixture must be null-free, which is WHY Arrow accepted it"
);
}
#[test]
fn uuid_extension_metadata_dropped_or_altered_is_rejected_and_arrow_would_accept_it() {
let (columns, rows) = uuid_and_text_columns();
let built = build_arrow_schema(&columns).expect("schema");
assert_eq!(
built
.field(0)
.metadata()
.get("ARROW:extension:name")
.map(String::as_str),
Some("arrow.uuid"),
"the fixture's uuid column must actually carry the extension metadata, or \
neither half below is a difference"
);
let stripped = schema_with(&columns, |i, f| {
if i == 0 {
f.with_metadata(HashMap::new())
} else {
f
}
});
let msg = expect_schema_mismatch(rows_to_record_batch_with_schema(
Arc::clone(&stripped),
&columns,
&rows,
));
assert!(
msg.contains("field 0 'id'") && msg.contains("metadata []") && msg.contains("arrow.uuid"),
"the message must name the position and both metadata sets, got: {msg}"
);
let accepted = try_new_accepts(stripped, &columns, &rows);
assert!(
accepted.schema().field(0).metadata().is_empty(),
"Arrow accepted a batch whose uuid column has NO extension metadata — a \
Parquet consumer of it loses the UUID logical type"
);
let altered = schema_with(&columns, |i, f| {
if i == 0 {
f.with_metadata(HashMap::from([(
"ARROW:extension:name".to_string(),
"arrow.not_a_uuid".to_string(),
)]))
} else {
f
}
});
let msg = expect_schema_mismatch(rows_to_record_batch_with_schema(
Arc::clone(&altered),
&columns,
&rows,
));
assert!(
msg.contains("arrow.not_a_uuid") && msg.contains("arrow.uuid"),
"the message must show both extension names, got: {msg}"
);
let accepted = try_new_accepts(altered, &columns, &rows);
assert_eq!(
accepted
.schema()
.field(0)
.metadata()
.get("ARROW:extension:name")
.map(String::as_str),
Some("arrow.not_a_uuid"),
"Arrow accepted the batch with a foreign extension name"
);
}
#[test]
fn extra_schema_level_metadata_is_rejected_and_arrow_would_accept_it() {
let (columns, rows) = two_text_columns();
assert!(
build_arrow_schema(&columns)
.expect("schema")
.metadata()
.is_empty(),
"build_arrow_schema must set no schema metadata, or this axis is not a \
difference"
);
let tagged = Arc::new(build_arrow_schema(&columns).expect("schema").with_metadata(
HashMap::from([("origin".to_string(), "elsewhere".to_string())]),
));
let msg = expect_schema_mismatch(rows_to_record_batch_with_schema(
Arc::clone(&tagged),
&columns,
&rows,
));
assert!(
msg.contains("top-level metadata") && msg.contains("origin"),
"the message must name the offending metadata, got: {msg}"
);
let accepted = try_new_accepts(tagged, &columns, &rows);
assert_eq!(
accepted
.schema()
.metadata()
.get("origin")
.map(String::as_str),
Some("elsewhere"),
"Arrow accepted a batch labelled with metadata the columns never produced"
);
}
#[test]
fn a_differing_datatype_is_rejected_here_with_a_named_axis_before_arrow_sees_it() {
let (columns, rows) = two_text_columns();
let retyped = schema_with(&columns, |i, f| {
if i == 1 {
Field::new(f.name(), arrow::datatypes::DataType::Int64, f.is_nullable())
} else {
f
}
});
let msg = expect_schema_mismatch(rows_to_record_batch_with_schema(
Arc::clone(&retyped),
&columns,
&rows,
));
assert!(
msg.contains("field 1 'beta'") && msg.contains("Int64") && msg.contains("Utf8"),
"the message must name the position and both Arrow types, got: {msg}"
);
let arrays = convert_to_arrays(&columns, &rows).expect("arrays");
let arrow_err = RecordBatch::try_new(retyped, arrays)
.expect_err("Arrow compares field data types, so it refuses this as well");
assert!(
!arrow_err.to_string().contains("column 1 is"),
"Arrow's message is the opaque one this check front-runs, got: {arrow_err}"
);
}
#[test]
fn a_matching_schema_with_uuid_extension_metadata_is_accepted() {
let (columns, rows) = uuid_and_text_columns();
let schema = Arc::new(build_arrow_schema(&columns).expect("schema"));
let batch = rows_to_record_batch_with_schema(Arc::clone(&schema), &columns, &rows)
.expect("the schema build_arrow_schema produced must be accepted");
assert_eq!(
batch.schema(),
schema,
"the batch keeps the supplied schema"
);
assert_eq!(
batch
.schema()
.field(0)
.metadata()
.get("ARROW:extension:name")
.map(String::as_str),
Some("arrow.uuid")
);
for _ in 0..3 {
rows_to_record_batch_with_schema(Arc::clone(&schema), &columns, &rows)
.expect("the same schema must be accepted for every batch of a scan");
}
}
#[test]
fn the_trusted_path_does_not_revalidate_and_the_external_one_still_does() {
let (columns, rows) = uuid_and_text_columns();
let before = super::schema_validations_on_this_thread();
for _ in 0..3 {
rows_to_record_batch(&columns, &rows).expect("inline schema must build");
}
assert_eq!(
super::schema_validations_on_this_thread() - before,
0,
"rows_to_record_batch must not revalidate the schema it just built with \
build_arrow_schema — that reconstructs every expected Field a second time, \
per batch"
);
let schema = Arc::new(build_arrow_schema(&columns).expect("schema"));
let before = super::schema_validations_on_this_thread();
for _ in 0..3 {
rows_to_record_batch_with_schema(Arc::clone(&schema), &columns, &rows)
.expect("the matching schema must be accepted");
}
assert_eq!(
super::schema_validations_on_this_thread() - before,
3,
"a caller-supplied schema must still be validated on every call — that is \
the documented public contract"
);
}
#[test]
fn the_trusted_path_returns_the_same_batch_as_the_validating_path() {
let (columns, rows) = uuid_and_text_columns();
let trusted = rows_to_record_batch(&columns, &rows).expect("inline schema");
let validated = rows_to_record_batch_with_schema(
Arc::new(build_arrow_schema(&columns).expect("schema")),
&columns,
&rows,
)
.expect("supplied schema");
assert_eq!(
trusted.schema(),
validated.schema(),
"schemas must be equal"
);
assert_eq!(trusted.num_rows(), validated.num_rows());
assert_eq!(trusted.num_columns(), validated.num_columns());
assert_eq!(
trusted
.schema()
.field(0)
.metadata()
.get("ARROW:extension:name")
.map(String::as_str),
Some("arrow.uuid"),
"the trusted path must keep the uuid extension metadata"
);
for i in 0..trusted.num_columns() {
assert_eq!(
trusted.column(i).to_data(),
validated.column(i).to_data(),
"column {i} must be byte-identical on both paths"
);
}
}
#[test]
fn arrow_refuses_a_zero_column_batch_unless_given_an_explicit_row_count() {
use arrow::record_batch::RecordBatchOptions;
let schema = Arc::new(Schema::empty());
let err = RecordBatch::try_new(Arc::clone(&schema), vec![])
.expect_err("arrow 53.4.1 refuses a zero-column batch with no explicit row count");
assert_eq!(
err.to_string(),
"Invalid argument error: must either specify a row count or at least one column"
);
for n in [0usize, 3] {
let batch = RecordBatch::try_new_with_options(
Arc::clone(&schema),
vec![],
&RecordBatchOptions::new().with_row_count(Some(n)),
)
.expect("an explicit row count makes a zero-column batch constructible");
assert_eq!(batch.num_rows(), n);
assert_eq!(batch.num_columns(), 0);
}
let no_columns: Vec<ColumnInfo> = Vec::new();
let rows: Vec<QueryRow> = (0..3).map(|_| row_one("a", Value::Integer(1))).collect();
let err = rows_to_record_batch(&no_columns, &rows)
.expect_err("rows_to_record_batch inherits arrow's refusal");
assert_eq!(
err.to_string(),
"Arrow error: Invalid argument error: must either specify a row count or at least one column"
);
}
#[test]
fn issue_4114_vector_column_exports_as_list_of_float32() {
let vec_ty = CqlType::Vector(Box::new(CqlType::Float), 3);
let columns = vec![col("v", DataType::List, Some(vec_ty.clone()))];
let rows = vec![row_one(
"v",
Value::List(vec![
Value::Float32(1.0),
Value::Float32(2.5),
Value::Float32(-3.75),
]),
)];
let batch = rows_to_record_batch(&columns, &rows).expect("a vector column must export");
let declared = cql_type_to_arrow_data_type(&vec_ty);
let actual = batch.column(0).data_type().clone();
assert_eq!(
actual, declared,
"the produced array type must equal the DECLARED schema type; a mismatch \
is an invalid RecordBatch (issue #4114, roborev job 110)"
);
match &actual {
ArrowDataType::List(item) => assert_eq!(
item.data_type(),
&ArrowDataType::Float32,
"vector<float, n> elements must be Float32, got {:?}",
item.data_type()
),
other => panic!("vector must map to an Arrow List, got {other:?}"),
}
assert_eq!(batch.num_rows(), 1, "the row must survive the export");
}
#[test]
fn issue_4114_vector_element_type_is_honoured_not_hardcoded() {
let vec_ty = CqlType::Vector(Box::new(CqlType::Double), 2);
let columns = vec![col("v", DataType::List, Some(vec_ty.clone()))];
let rows = vec![row_one(
"v",
Value::List(vec![Value::Float(1.5), Value::Float(-2.25)]),
)];
let batch = rows_to_record_batch(&columns, &rows).expect("a double vector must export");
assert_eq!(
batch.column(0).data_type(),
&cql_type_to_arrow_data_type(&vec_ty),
"array type must track the DECLARED element type"
);
match batch.column(0).data_type() {
ArrowDataType::List(item) => assert_eq!(item.data_type(), &ArrowDataType::Float64),
other => panic!("expected a List, got {other:?}"),
}
}