vortex_array/scalar_fn/fns/
byte_length.rs1use num_traits::AsPrimitive;
5use vortex_buffer::Buffer;
6use vortex_error::VortexExpect;
7use vortex_error::VortexResult;
8use vortex_error::vortex_bail;
9use vortex_error::vortex_ensure_eq;
10use vortex_session::VortexSession;
11use vortex_session::registry::CachedId;
12
13use crate::ArrayRef;
14use crate::ExecutionCtx;
15use crate::IntoArray;
16use crate::array::ArrayView;
17use crate::array::VTable;
18use crate::arrays::ConstantArray;
19use crate::arrays::PrimitiveArray;
20use crate::arrays::VarBinViewArray;
21use crate::arrays::scalar_fn::ExactScalarFn;
22use crate::arrays::scalar_fn::ScalarFnArrayView;
23use crate::arrays::varbinview::VarBinViewArrayExt;
24use crate::dtype::DType;
25use crate::dtype::Nullability;
26use crate::dtype::PType;
27use crate::kernel::ExecuteParentKernel;
28use crate::scalar::Scalar;
29use crate::scalar_fn::Arity;
30use crate::scalar_fn::ChildName;
31use crate::scalar_fn::EmptyOptions;
32use crate::scalar_fn::ExecutionArgs;
33use crate::scalar_fn::ScalarFnId;
34use crate::scalar_fn::ScalarFnVTable;
35
36pub trait ByteLengthKernel: VTable {
37 fn byte_length(
38 array: ArrayView<'_, Self>,
39 ctx: &mut ExecutionCtx,
40 ) -> VortexResult<Option<ArrayRef>>;
41}
42
43#[derive(Default, Debug)]
44pub struct ByteLengthExecuteAdaptor<V>(pub V);
45
46impl<V: ByteLengthKernel> ExecuteParentKernel<V> for ByteLengthExecuteAdaptor<V> {
47 type Parent = ExactScalarFn<ByteLength>;
48
49 fn execute_parent(
50 &self,
51 array: ArrayView<'_, V>,
52 _parent: ScalarFnArrayView<'_, ByteLength>,
53 child_idx: usize,
54 ctx: &mut ExecutionCtx,
55 ) -> VortexResult<Option<ArrayRef>> {
56 vortex_ensure_eq!(child_idx, 0);
57 V::byte_length(array, ctx)
58 }
59}
60
61#[derive(Clone)]
63pub struct ByteLength;
64
65impl ScalarFnVTable for ByteLength {
66 type Options = EmptyOptions;
67
68 fn id(&self) -> ScalarFnId {
69 static ID: CachedId = CachedId::new("vortex.byte_length");
70 *ID
71 }
72
73 fn serialize(&self, _instance: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
74 Ok(Some(vec![]))
75 }
76
77 fn deserialize(
78 &self,
79 _metadata: &[u8],
80 _session: &VortexSession,
81 ) -> VortexResult<Self::Options> {
82 Ok(EmptyOptions)
83 }
84
85 fn arity(&self, _options: &Self::Options) -> Arity {
86 Arity::Exact(1)
87 }
88
89 fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName {
90 match child_idx {
91 0 => ChildName::from("input"),
92 _ => unreachable!("Invalid child index {child_idx} for byte_length()"),
93 }
94 }
95
96 fn return_dtype(&self, _options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
97 match &arg_dtypes[0] {
98 DType::Utf8(nullable) | DType::Binary(nullable) => {
99 Ok(DType::Primitive(PType::U64, *nullable))
100 }
101 other => {
102 vortex_bail!("byte_length() requires Utf8 or Binary, got {other}")
103 }
104 }
105 }
106
107 fn execute(
108 &self,
109 _options: &Self::Options,
110 args: &dyn ExecutionArgs,
111 ctx: &mut ExecutionCtx,
112 ) -> VortexResult<ArrayRef> {
113 let input = args.get(0)?;
114 let nullability = input.dtype().nullability();
115
116 if let Some(scalar) = input.as_constant() {
117 let len_scalar = scalar_byte_length(scalar, nullability)?;
118 return Ok(ConstantArray::new(len_scalar, args.row_count()).into_array());
119 }
120
121 match input.dtype() {
122 DType::Utf8(_) | DType::Binary(_) => byte_length(&input, nullability, ctx),
123 other => {
124 vortex_bail!("byte_length() requires Utf8 or Binary, got {other}")
125 }
126 }
127 }
128
129 fn is_strict(&self, _options: &Self::Options) -> bool {
130 true
131 }
132
133 fn is_infallible(&self, _options: &Self::Options) -> bool {
134 true
135 }
136}
137
138fn scalar_byte_length(scalar: &Scalar, nullability: Nullability) -> VortexResult<Scalar> {
139 if scalar.is_null() {
140 let dtype = DType::Primitive(PType::U64, Nullability::Nullable);
141 return Ok(Scalar::null(dtype));
142 }
143 let len = match scalar.dtype() {
144 DType::Utf8(_) => scalar
145 .as_utf8()
146 .value()
147 .vortex_expect("null utf-8 scalar")
148 .len(),
149 DType::Binary(_) => scalar
150 .as_binary()
151 .value()
152 .vortex_expect("null binary scalar")
153 .len(),
154 other => {
155 vortex_bail!("byte_length() requires Utf8 or Binary, got {other}")
156 }
157 };
158 let len: u64 = len.as_();
159 Ok(Scalar::primitive(len, nullability))
160}
161
162pub(crate) fn byte_length(
163 array: &ArrayRef,
164 nullability: Nullability,
165 ctx: &mut ExecutionCtx,
166) -> VortexResult<ArrayRef> {
167 let array = array.clone().execute::<VarBinViewArray>(ctx)?;
168 let validity = array.varbinview_validity();
169 let lengths: Buffer<u64> = array.views().iter().map(|v| v.len() as u64).collect();
170 Ok(PrimitiveArray::new(lengths, validity.union_nullability(nullability)).into_array())
171}
172
173#[cfg(test)]
174mod tests {
175 use std::sync::LazyLock;
176
177 use rstest::rstest;
178 use vortex_error::VortexResult;
179 use vortex_session::VortexSession;
180
181 use crate::ArrayRef;
182 use crate::IntoArray;
183 use crate::VortexSessionExecute;
184 use crate::arrays::ConstantArray;
185 use crate::arrays::PrimitiveArray;
186 use crate::arrays::VarBinArray;
187 use crate::arrays::VarBinViewArray;
188 use crate::assert_arrays_eq;
189 use crate::dtype::DType;
190 use crate::dtype::Nullability;
191 use crate::expr::byte_length;
192 use crate::expr::root;
193 use crate::scalar::Scalar;
194
195 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
196
197 #[rstest]
198 #[case(VarBinArray::from_strs(vec!["hello", "world", ""]).into_array(), vec![5u64, 5, 0])]
199 #[case(VarBinArray::from_bytes(vec![b"ab".as_ref(), b"cde"]).into_array(), vec![2u64, 3])]
200 #[case(VarBinArray::from_strs(vec!["Пуховички"]).into_array(), vec![18u64])]
201 #[case(VarBinArray::from_bytes(vec!["Пуховички".as_ref()]).into_array(), vec![18u64])]
202 fn test_bytes_byte_length(
203 #[case] array: ArrayRef,
204 #[case] expected_lens: Vec<u64>,
205 ) -> VortexResult<()> {
206 let mut ctx = SESSION.create_execution_ctx();
207 let result = array.apply(&byte_length(root()))?;
208 let expected = PrimitiveArray::from_iter(expected_lens);
209 assert_arrays_eq!(result, expected, &mut ctx);
210 Ok(())
211 }
212
213 #[test]
214 fn test_varbinview_byte_length() -> VortexResult<()> {
215 let mut ctx = SESSION.create_execution_ctx();
216 let array = VarBinViewArray::from_iter_str(["short", "a longer string here"]).into_array();
217 let result = array.apply(&byte_length(root()))?;
218 let expected = PrimitiveArray::from_iter(vec![5u64, 20]);
219 assert_arrays_eq!(result, expected, &mut ctx);
220 Ok(())
221 }
222
223 #[test]
224 fn test_nullable_string_byte_length() -> VortexResult<()> {
225 let array = VarBinArray::from_nullable_strs(vec![Some("hello"), None, Some("Пуховички")])
226 .into_array();
227 let result = array.apply(&byte_length(root()))?;
228
229 let mut ctx = SESSION.create_execution_ctx();
230 assert!(result.is_valid(0, &mut ctx)?);
231 assert!(!result.is_valid(1, &mut ctx)?);
232 assert!(result.is_valid(2, &mut ctx)?);
233 assert_eq!(
234 result.execute_scalar(0, &mut SESSION.create_execution_ctx())?,
235 Scalar::primitive(5u64, Nullability::Nullable),
236 );
237 assert_eq!(
238 result.execute_scalar(2, &mut SESSION.create_execution_ctx())?,
239 Scalar::primitive(18u64, Nullability::Nullable),
240 );
241 assert_arrays_eq!(
242 result,
243 PrimitiveArray::from_option_iter([Some(5u64), None, Some(18)]),
244 &mut ctx
245 );
246 Ok(())
247 }
248
249 #[test]
250 fn test_null_scalar_byte_length() -> VortexResult<()> {
251 let null_scalar = Scalar::null(DType::Utf8(Nullability::Nullable));
252 let array = ConstantArray::new(null_scalar, 2).into_array();
253 let result = array.apply(&byte_length(root()))?;
254 let mut ctx = SESSION.create_execution_ctx();
255 assert!(!result.is_valid(0, &mut ctx)?);
256 assert!(!result.is_valid(1, &mut ctx)?);
257 assert_arrays_eq!(
258 result,
259 PrimitiveArray::from_option_iter([None::<u64>, None]),
260 &mut ctx
261 );
262 Ok(())
263 }
264
265 #[test]
266 fn test_display() {
267 let expr = byte_length(root());
268 assert_eq!(expr.to_string(), "vortex.byte_length($)");
269 }
270}