1use std::io::Write;
10use std::str::FromStr;
11use std::sync::{Arc, LazyLock};
12
13use datafusion::arrow::array::{Array, ArrowPrimitiveType, AsArray, PrimitiveArray, RecordBatch};
14use datafusion::arrow::datatypes::{
15 DataType, FieldRef, Float16Type, Float32Type, Float64Type, SchemaRef,
16};
17use datafusion::arrow::error::ArrowError;
18use datafusion::arrow::json::writer::{
19 Encoder, EncoderFactory, EncoderOptions, JsonArray, NullableEncoder, WriterBuilder,
20 make_encoder,
21};
22use datafusion::arrow::util::pretty::pretty_format_batches;
23use oxidelake_core::EngineError;
24
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
27#[non_exhaustive]
28pub enum OutputFormat {
29 #[default]
31 Table,
32 Json,
36 Csv,
38}
39
40impl OutputFormat {
41 pub const ALL: [OutputFormat; 3] = [OutputFormat::Table, OutputFormat::Json, OutputFormat::Csv];
43
44 pub const fn as_str(self) -> &'static str {
46 match self {
47 OutputFormat::Table => "table",
48 OutputFormat::Json => "json",
49 OutputFormat::Csv => "csv",
50 }
51 }
52}
53
54impl std::fmt::Display for OutputFormat {
55 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56 f.write_str(self.as_str())
57 }
58}
59
60impl FromStr for OutputFormat {
61 type Err = EngineError;
62
63 fn from_str(s: &str) -> Result<Self, Self::Err> {
64 match s.trim().to_ascii_lowercase().as_str() {
65 "table" => Ok(OutputFormat::Table),
66 "json" => Ok(OutputFormat::Json),
67 "csv" => Ok(OutputFormat::Csv),
68 other => Err(EngineError::plan(format!(
69 "unknown output format '{other}'; expected one of table, json, csv"
70 ))),
71 }
72 }
73}
74
75pub fn render(
81 schema: &SchemaRef,
82 batches: &[RecordBatch],
83 format: OutputFormat,
84) -> Result<String, EngineError> {
85 match format {
86 OutputFormat::Table => Ok(format!("{}\n", pretty_format_batches(batches)?)),
87 OutputFormat::Json => {
88 let mut buffer = Vec::new();
89 let mut writer = WriterBuilder::new()
94 .with_explicit_nulls(true)
95 .with_encoder_factory(Arc::new(NonFiniteFloats))
96 .build::<_, JsonArray>(&mut buffer);
97 writer.write_batches(&batches.iter().collect::<Vec<_>>())?;
98 writer.finish()?;
99 finish(buffer)
100 }
101 OutputFormat::Csv => {
102 let mut buffer = Vec::new();
103 let mut writer = datafusion::arrow::csv::Writer::new(&mut buffer);
104 let empty;
105 let batches = if batches.is_empty() {
106 empty = [RecordBatch::new_empty(SchemaRef::clone(schema))];
107 &empty[..]
108 } else {
109 batches
110 };
111 for batch in batches {
112 writer.write(batch)?;
113 }
114 drop(writer);
115 finish(buffer)
116 }
117 }
118}
119
120static PLAIN: LazyLock<EncoderOptions> = LazyLock::new(EncoderOptions::default);
124
125#[derive(Debug)]
136struct NonFiniteFloats;
137
138impl EncoderFactory for NonFiniteFloats {
139 fn make_default_encoder<'a>(
140 &self,
141 field: &'a FieldRef,
142 array: &'a dyn Array,
143 _options: &'a EncoderOptions,
144 ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
145 let encoder: Box<dyn Encoder + 'a> = match array.data_type() {
146 DataType::Float16 => Box::new(FloatEncoder::new(
147 array.as_primitive::<Float16Type>(),
148 field,
149 |v| v.to_f64(),
150 )?),
151 DataType::Float32 => Box::new(FloatEncoder::new(
152 array.as_primitive::<Float32Type>(),
153 field,
154 f64::from,
155 )?),
156 DataType::Float64 => Box::new(FloatEncoder::new(
157 array.as_primitive::<Float64Type>(),
158 field,
159 |v| v,
160 )?),
161 _ => return Ok(None),
162 };
163 Ok(Some(NullableEncoder::new(encoder, array.nulls().cloned())))
164 }
165}
166
167struct FloatEncoder<'a, T: ArrowPrimitiveType> {
170 array: &'a PrimitiveArray<T>,
171 to_f64: fn(T::Native) -> f64,
172 finite: NullableEncoder<'a>,
173}
174
175impl<'a, T: ArrowPrimitiveType> FloatEncoder<'a, T> {
176 fn new(
177 array: &'a PrimitiveArray<T>,
178 field: &'a FieldRef,
179 to_f64: fn(T::Native) -> f64,
180 ) -> Result<Self, ArrowError> {
181 Ok(Self {
182 array,
183 to_f64,
184 finite: make_encoder(field, array, &PLAIN)?,
185 })
186 }
187}
188
189impl<T: ArrowPrimitiveType> Encoder for FloatEncoder<'_, T> {
190 fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
191 let value = (self.to_f64)(self.array.value(idx));
192 if value.is_nan() {
193 out.extend_from_slice(b"\"NaN\"");
194 } else if value == f64::INFINITY {
195 out.extend_from_slice(b"\"Infinity\"");
196 } else if value == f64::NEG_INFINITY {
197 out.extend_from_slice(b"\"-Infinity\"");
198 } else {
199 self.finite.encode(idx, out);
200 }
201 }
202}
203
204fn finish(mut buffer: Vec<u8>) -> Result<String, EngineError> {
206 if !buffer.ends_with(b"\n") {
207 buffer.write_all(b"\n").map_err(EngineError::from)?;
208 }
209 String::from_utf8(buffer)
210 .map_err(|e| EngineError::execution(format!("rendered output was not UTF-8: {e}")))
211}
212
213#[cfg(test)]
214#[allow(clippy::unwrap_used, clippy::expect_used)]
215mod tests {
216 use std::sync::Arc;
217
218 use datafusion::arrow::array::{
219 ArrayRef, DictionaryArray, FixedSizeListArray, Float32Array, Float64Array, Int8Array,
220 Int64Array, StringArray, StructArray,
221 };
222 use datafusion::arrow::compute::cast;
223 use datafusion::arrow::datatypes::{DataType, Field, Int8Type, Schema};
224
225 use super::*;
226
227 fn schema() -> SchemaRef {
228 Arc::new(Schema::new(vec![
229 Field::new("k", DataType::Int64, true),
230 Field::new("name", DataType::Utf8, true),
231 ]))
232 }
233
234 fn batch() -> RecordBatch {
235 RecordBatch::try_new(
236 schema(),
237 vec![
238 Arc::new(Int64Array::from(vec![Some(1), None])),
239 Arc::new(StringArray::from(vec![Some("a,b"), None])),
240 ],
241 )
242 .unwrap()
243 }
244
245 #[test]
246 fn every_name_round_trips() {
247 for format in OutputFormat::ALL {
248 assert_eq!(format.as_str().parse::<OutputFormat>().unwrap(), format);
249 }
250 assert_eq!(" CSV ".parse::<OutputFormat>().unwrap(), OutputFormat::Csv);
251 }
252
253 #[test]
254 fn an_unknown_name_names_the_alternatives() {
255 let message = "yaml".parse::<OutputFormat>().unwrap_err().to_string();
256 assert!(message.contains("yaml"), "{message}");
257 assert!(message.contains("table, json, csv"), "{message}");
258 }
259
260 #[test]
264 fn json_is_an_array_of_objects_with_every_column() {
265 let text = render(&schema(), &[batch()], OutputFormat::Json).unwrap();
266 assert_eq!(
267 text,
268 "[{\"k\":1,\"name\":\"a,b\"},{\"k\":null,\"name\":null}]\n"
269 );
270 }
271
272 #[test]
277 fn json_writes_nan_and_the_infinities_as_strings() {
278 let item = Arc::new(Field::new("item", DataType::Float32, true));
279 let schema = Arc::new(Schema::new(vec![
280 Field::new("d", DataType::Float64, true),
281 Field::new("f", DataType::Float32, true),
282 Field::new("h", DataType::Float16, true),
283 Field::new("v", DataType::FixedSizeList(Arc::clone(&item), 2), true),
284 ]));
285 let d = Float64Array::from(vec![
286 Some(f64::NAN),
287 Some(f64::INFINITY),
288 Some(f64::NEG_INFINITY),
289 Some(1.5),
290 None,
291 ]);
292 let f = Float32Array::from(vec![
293 Some(f32::NEG_INFINITY),
294 Some(f32::NAN),
295 Some(0.25),
296 Some(f32::INFINITY),
297 None,
298 ]);
299 let h = cast(
302 &Float32Array::from(vec![
303 Some(0.5),
304 Some(f32::NEG_INFINITY),
305 Some(f32::NAN),
306 Some(f32::INFINITY),
307 None,
308 ]),
309 &DataType::Float16,
310 )
311 .unwrap();
312 let v = FixedSizeListArray::new(
313 item,
314 2,
315 Arc::new(Float32Array::from(vec![
316 Some(f32::NAN),
317 Some(0.5),
318 None,
319 Some(f32::INFINITY),
320 Some(0.0),
321 Some(0.0),
322 Some(0.0),
323 Some(0.0),
324 Some(0.0),
325 Some(0.0),
326 ])),
327 Some(vec![true, true, true, true, false].into()),
328 );
329 let batch = RecordBatch::try_new(
330 Arc::clone(&schema),
331 vec![Arc::new(d), Arc::new(f), h, Arc::new(v)],
332 )
333 .unwrap();
334 let text = render(&schema, &[batch], OutputFormat::Json).unwrap();
335 assert_eq!(
336 text,
337 concat!(
338 "[{\"d\":\"NaN\",\"f\":\"-Infinity\",\"h\":0.5,\"v\":[\"NaN\",0.5]},",
339 "{\"d\":\"Infinity\",\"f\":\"NaN\",\"h\":\"-Infinity\",\"v\":[null,\"Infinity\"]},",
340 "{\"d\":\"-Infinity\",\"f\":0.25,\"h\":\"NaN\",\"v\":[0.0,0.0]},",
341 "{\"d\":1.5,\"f\":\"Infinity\",\"h\":\"Infinity\",\"v\":[0.0,0.0]},",
342 "{\"d\":null,\"f\":null,\"h\":null,\"v\":null}]\n",
343 )
344 );
345 }
346
347 #[test]
351 fn json_reaches_floats_inside_structs_and_dictionaries() {
352 let x = Arc::new(Float64Array::from(vec![Some(f64::NAN), None])) as ArrayRef;
353 let s = StructArray::from(vec![(
354 Arc::new(Field::new("x", DataType::Float64, true)),
355 x,
356 )]);
357 let e = DictionaryArray::<Int8Type>::try_new(
358 Int8Array::from(vec![1, 0]),
359 Arc::new(Float64Array::from(vec![f64::NEG_INFINITY, 2.5])),
360 )
361 .unwrap();
362 let schema = Arc::new(Schema::new(vec![
363 Field::new("s", s.data_type().clone(), true),
364 Field::new("e", e.data_type().clone(), true),
365 ]));
366 let batch =
367 RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(s), Arc::new(e)]).unwrap();
368 let text = render(&schema, &[batch], OutputFormat::Json).unwrap();
369 assert_eq!(
370 text,
371 "[{\"s\":{\"x\":\"NaN\"},\"e\":2.5},{\"s\":{\"x\":null},\"e\":\"-Infinity\"}]\n"
372 );
373 }
374
375 #[test]
376 fn csv_quotes_an_embedded_comma_and_keeps_the_header() {
377 let text = render(&schema(), &[batch()], OutputFormat::Csv).unwrap();
378 assert_eq!(text, "k,name\n1,\"a,b\"\n,\n");
379 }
380
381 #[test]
384 fn an_empty_result_still_has_a_csv_header() {
385 assert_eq!(
386 render(&schema(), &[], OutputFormat::Csv).unwrap(),
387 "k,name\n"
388 );
389 assert_eq!(render(&schema(), &[], OutputFormat::Json).unwrap(), "[]\n");
390 }
391
392 #[test]
393 fn the_table_form_is_the_default() {
394 assert_eq!(OutputFormat::default(), OutputFormat::Table);
395 let text = render(&schema(), &[batch()], OutputFormat::Table).unwrap();
396 assert!(text.contains("| k "), "{text}");
397 assert!(text.ends_with("\n"), "{text}");
398 }
399}