use std::io::Write;
use std::str::FromStr;
use std::sync::{Arc, LazyLock};
use datafusion::arrow::array::{Array, ArrowPrimitiveType, AsArray, PrimitiveArray, RecordBatch};
use datafusion::arrow::datatypes::{
DataType, FieldRef, Float16Type, Float32Type, Float64Type, SchemaRef,
};
use datafusion::arrow::error::ArrowError;
use datafusion::arrow::json::writer::{
Encoder, EncoderFactory, EncoderOptions, JsonArray, NullableEncoder, WriterBuilder,
make_encoder,
};
use datafusion::arrow::util::pretty::pretty_format_batches;
use oxidelake_core::EngineError;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum OutputFormat {
#[default]
Table,
Json,
Csv,
}
impl OutputFormat {
pub const ALL: [OutputFormat; 3] = [OutputFormat::Table, OutputFormat::Json, OutputFormat::Csv];
pub const fn as_str(self) -> &'static str {
match self {
OutputFormat::Table => "table",
OutputFormat::Json => "json",
OutputFormat::Csv => "csv",
}
}
}
impl std::fmt::Display for OutputFormat {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for OutputFormat {
type Err = EngineError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.trim().to_ascii_lowercase().as_str() {
"table" => Ok(OutputFormat::Table),
"json" => Ok(OutputFormat::Json),
"csv" => Ok(OutputFormat::Csv),
other => Err(EngineError::plan(format!(
"unknown output format '{other}'; expected one of table, json, csv"
))),
}
}
}
pub fn render(
schema: &SchemaRef,
batches: &[RecordBatch],
format: OutputFormat,
) -> Result<String, EngineError> {
match format {
OutputFormat::Table => Ok(format!("{}\n", pretty_format_batches(batches)?)),
OutputFormat::Json => {
let mut buffer = Vec::new();
let mut writer = WriterBuilder::new()
.with_explicit_nulls(true)
.with_encoder_factory(Arc::new(NonFiniteFloats))
.build::<_, JsonArray>(&mut buffer);
writer.write_batches(&batches.iter().collect::<Vec<_>>())?;
writer.finish()?;
finish(buffer)
}
OutputFormat::Csv => {
let mut buffer = Vec::new();
let mut writer = datafusion::arrow::csv::Writer::new(&mut buffer);
let empty;
let batches = if batches.is_empty() {
empty = [RecordBatch::new_empty(SchemaRef::clone(schema))];
&empty[..]
} else {
batches
};
for batch in batches {
writer.write(batch)?;
}
drop(writer);
finish(buffer)
}
}
}
static PLAIN: LazyLock<EncoderOptions> = LazyLock::new(EncoderOptions::default);
#[derive(Debug)]
struct NonFiniteFloats;
impl EncoderFactory for NonFiniteFloats {
fn make_default_encoder<'a>(
&self,
field: &'a FieldRef,
array: &'a dyn Array,
_options: &'a EncoderOptions,
) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
let encoder: Box<dyn Encoder + 'a> = match array.data_type() {
DataType::Float16 => Box::new(FloatEncoder::new(
array.as_primitive::<Float16Type>(),
field,
|v| v.to_f64(),
)?),
DataType::Float32 => Box::new(FloatEncoder::new(
array.as_primitive::<Float32Type>(),
field,
f64::from,
)?),
DataType::Float64 => Box::new(FloatEncoder::new(
array.as_primitive::<Float64Type>(),
field,
|v| v,
)?),
_ => return Ok(None),
};
Ok(Some(NullableEncoder::new(encoder, array.nulls().cloned())))
}
}
struct FloatEncoder<'a, T: ArrowPrimitiveType> {
array: &'a PrimitiveArray<T>,
to_f64: fn(T::Native) -> f64,
finite: NullableEncoder<'a>,
}
impl<'a, T: ArrowPrimitiveType> FloatEncoder<'a, T> {
fn new(
array: &'a PrimitiveArray<T>,
field: &'a FieldRef,
to_f64: fn(T::Native) -> f64,
) -> Result<Self, ArrowError> {
Ok(Self {
array,
to_f64,
finite: make_encoder(field, array, &PLAIN)?,
})
}
}
impl<T: ArrowPrimitiveType> Encoder for FloatEncoder<'_, T> {
fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
let value = (self.to_f64)(self.array.value(idx));
if value.is_nan() {
out.extend_from_slice(b"\"NaN\"");
} else if value == f64::INFINITY {
out.extend_from_slice(b"\"Infinity\"");
} else if value == f64::NEG_INFINITY {
out.extend_from_slice(b"\"-Infinity\"");
} else {
self.finite.encode(idx, out);
}
}
}
fn finish(mut buffer: Vec<u8>) -> Result<String, EngineError> {
if !buffer.ends_with(b"\n") {
buffer.write_all(b"\n").map_err(EngineError::from)?;
}
String::from_utf8(buffer)
.map_err(|e| EngineError::execution(format!("rendered output was not UTF-8: {e}")))
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use std::sync::Arc;
use datafusion::arrow::array::{
ArrayRef, DictionaryArray, FixedSizeListArray, Float32Array, Float64Array, Int8Array,
Int64Array, StringArray, StructArray,
};
use datafusion::arrow::compute::cast;
use datafusion::arrow::datatypes::{DataType, Field, Int8Type, Schema};
use super::*;
fn schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("k", DataType::Int64, true),
Field::new("name", DataType::Utf8, true),
]))
}
fn batch() -> RecordBatch {
RecordBatch::try_new(
schema(),
vec![
Arc::new(Int64Array::from(vec![Some(1), None])),
Arc::new(StringArray::from(vec![Some("a,b"), None])),
],
)
.unwrap()
}
#[test]
fn every_name_round_trips() {
for format in OutputFormat::ALL {
assert_eq!(format.as_str().parse::<OutputFormat>().unwrap(), format);
}
assert_eq!(" CSV ".parse::<OutputFormat>().unwrap(), OutputFormat::Csv);
}
#[test]
fn an_unknown_name_names_the_alternatives() {
let message = "yaml".parse::<OutputFormat>().unwrap_err().to_string();
assert!(message.contains("yaml"), "{message}");
assert!(message.contains("table, json, csv"), "{message}");
}
#[test]
fn json_is_an_array_of_objects_with_every_column() {
let text = render(&schema(), &[batch()], OutputFormat::Json).unwrap();
assert_eq!(
text,
"[{\"k\":1,\"name\":\"a,b\"},{\"k\":null,\"name\":null}]\n"
);
}
#[test]
fn json_writes_nan_and_the_infinities_as_strings() {
let item = Arc::new(Field::new("item", DataType::Float32, true));
let schema = Arc::new(Schema::new(vec![
Field::new("d", DataType::Float64, true),
Field::new("f", DataType::Float32, true),
Field::new("h", DataType::Float16, true),
Field::new("v", DataType::FixedSizeList(Arc::clone(&item), 2), true),
]));
let d = Float64Array::from(vec![
Some(f64::NAN),
Some(f64::INFINITY),
Some(f64::NEG_INFINITY),
Some(1.5),
None,
]);
let f = Float32Array::from(vec![
Some(f32::NEG_INFINITY),
Some(f32::NAN),
Some(0.25),
Some(f32::INFINITY),
None,
]);
let h = cast(
&Float32Array::from(vec![
Some(0.5),
Some(f32::NEG_INFINITY),
Some(f32::NAN),
Some(f32::INFINITY),
None,
]),
&DataType::Float16,
)
.unwrap();
let v = FixedSizeListArray::new(
item,
2,
Arc::new(Float32Array::from(vec![
Some(f32::NAN),
Some(0.5),
None,
Some(f32::INFINITY),
Some(0.0),
Some(0.0),
Some(0.0),
Some(0.0),
Some(0.0),
Some(0.0),
])),
Some(vec![true, true, true, true, false].into()),
);
let batch = RecordBatch::try_new(
Arc::clone(&schema),
vec![Arc::new(d), Arc::new(f), h, Arc::new(v)],
)
.unwrap();
let text = render(&schema, &[batch], OutputFormat::Json).unwrap();
assert_eq!(
text,
concat!(
"[{\"d\":\"NaN\",\"f\":\"-Infinity\",\"h\":0.5,\"v\":[\"NaN\",0.5]},",
"{\"d\":\"Infinity\",\"f\":\"NaN\",\"h\":\"-Infinity\",\"v\":[null,\"Infinity\"]},",
"{\"d\":\"-Infinity\",\"f\":0.25,\"h\":\"NaN\",\"v\":[0.0,0.0]},",
"{\"d\":1.5,\"f\":\"Infinity\",\"h\":\"Infinity\",\"v\":[0.0,0.0]},",
"{\"d\":null,\"f\":null,\"h\":null,\"v\":null}]\n",
)
);
}
#[test]
fn json_reaches_floats_inside_structs_and_dictionaries() {
let x = Arc::new(Float64Array::from(vec![Some(f64::NAN), None])) as ArrayRef;
let s = StructArray::from(vec![(
Arc::new(Field::new("x", DataType::Float64, true)),
x,
)]);
let e = DictionaryArray::<Int8Type>::try_new(
Int8Array::from(vec![1, 0]),
Arc::new(Float64Array::from(vec![f64::NEG_INFINITY, 2.5])),
)
.unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("s", s.data_type().clone(), true),
Field::new("e", e.data_type().clone(), true),
]));
let batch =
RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(s), Arc::new(e)]).unwrap();
let text = render(&schema, &[batch], OutputFormat::Json).unwrap();
assert_eq!(
text,
"[{\"s\":{\"x\":\"NaN\"},\"e\":2.5},{\"s\":{\"x\":null},\"e\":\"-Infinity\"}]\n"
);
}
#[test]
fn csv_quotes_an_embedded_comma_and_keeps_the_header() {
let text = render(&schema(), &[batch()], OutputFormat::Csv).unwrap();
assert_eq!(text, "k,name\n1,\"a,b\"\n,\n");
}
#[test]
fn an_empty_result_still_has_a_csv_header() {
assert_eq!(
render(&schema(), &[], OutputFormat::Csv).unwrap(),
"k,name\n"
);
assert_eq!(render(&schema(), &[], OutputFormat::Json).unwrap(), "[]\n");
}
#[test]
fn the_table_form_is_the_default() {
assert_eq!(OutputFormat::default(), OutputFormat::Table);
let text = render(&schema(), &[batch()], OutputFormat::Table).unwrap();
assert!(text.contains("| k "), "{text}");
assert!(text.ends_with("\n"), "{text}");
}
}