Skip to main content

oxidelake_runtime/
output.rs

1//! How `oxide sql` renders a result set (#49).
2//!
3//! The table form is for a person reading a terminal; `json` and `csv` exist
4//! so the same query can feed a script without a second tool to parse the box
5//! drawing. All three render the *same* batches — the format is a rendering
6//! choice made after execution, never a planning one, so a result cannot
7//! differ between them.
8
9use std::io::Write;
10use std::str::FromStr;
11use std::sync::{Arc, LazyLock};
12
13use datafusion::arrow::array::{Array, ArrowPrimitiveType, AsArray, PrimitiveArray, RecordBatch};
14use datafusion::arrow::datatypes::{
15    DataType, FieldRef, Float16Type, Float32Type, Float64Type, SchemaRef,
16};
17use datafusion::arrow::error::ArrowError;
18use datafusion::arrow::json::writer::{
19    Encoder, EncoderFactory, EncoderOptions, JsonArray, NullableEncoder, WriterBuilder,
20    make_encoder,
21};
22use datafusion::arrow::util::pretty::pretty_format_batches;
23use oxidelake_core::EngineError;
24
25/// Output formats accepted by `oxide sql --output`.
26#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
27#[non_exhaustive]
28pub enum OutputFormat {
29    /// An aligned ASCII table (the default).
30    #[default]
31    Table,
32    /// A JSON array of objects, one per row, each carrying every column: a
33    /// NULL is `null`, and NaN and the infinities, which JSON has no number
34    /// for, are the strings `"NaN"`, `"Infinity"` and `"-Infinity"`.
35    Json,
36    /// RFC 4180 CSV with a header row.
37    Csv,
38}
39
40impl OutputFormat {
41    /// Every format, in the order `--help` lists them.
42    pub const ALL: [OutputFormat; 3] = [OutputFormat::Table, OutputFormat::Json, OutputFormat::Csv];
43
44    /// The canonical lowercase name used on the command line.
45    pub const fn as_str(self) -> &'static str {
46        match self {
47            OutputFormat::Table => "table",
48            OutputFormat::Json => "json",
49            OutputFormat::Csv => "csv",
50        }
51    }
52}
53
54impl std::fmt::Display for OutputFormat {
55    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56        f.write_str(self.as_str())
57    }
58}
59
60impl FromStr for OutputFormat {
61    type Err = EngineError;
62
63    fn from_str(s: &str) -> Result<Self, Self::Err> {
64        match s.trim().to_ascii_lowercase().as_str() {
65            "table" => Ok(OutputFormat::Table),
66            "json" => Ok(OutputFormat::Json),
67            "csv" => Ok(OutputFormat::Csv),
68            other => Err(EngineError::plan(format!(
69                "unknown output format '{other}'; expected one of table, json, csv"
70            ))),
71        }
72    }
73}
74
75/// Renders `batches` in `format`, terminated by a newline.
76///
77/// `schema` is carried separately because an empty result has no batch to take
78/// it from, and a CSV reader handed nothing at all cannot tell an empty result
79/// from a failed one — so the header row is written even when no row is.
80pub fn render(
81    schema: &SchemaRef,
82    batches: &[RecordBatch],
83    format: OutputFormat,
84) -> Result<String, EngineError> {
85    match format {
86        OutputFormat::Table => Ok(format!("{}\n", pretty_format_batches(batches)?)),
87        OutputFormat::Json => {
88            let mut buffer = Vec::new();
89            // arrow-json leaves a NULL's key out of its object by default, so
90            // a script could not tell a NULL from a column that does not
91            // exist; and it writes NaN and both infinities as `null`, so it
92            // could not tell them from a NULL either (#124).
93            let mut writer = WriterBuilder::new()
94                .with_explicit_nulls(true)
95                .with_encoder_factory(Arc::new(NonFiniteFloats))
96                .build::<_, JsonArray>(&mut buffer);
97            writer.write_batches(&batches.iter().collect::<Vec<_>>())?;
98            writer.finish()?;
99            finish(buffer)
100        }
101        OutputFormat::Csv => {
102            let mut buffer = Vec::new();
103            let mut writer = datafusion::arrow::csv::Writer::new(&mut buffer);
104            let empty;
105            let batches = if batches.is_empty() {
106                empty = [RecordBatch::new_empty(SchemaRef::clone(schema))];
107                &empty[..]
108            } else {
109                batches
110            };
111            for batch in batches {
112                writer.write(batch)?;
113            }
114            drop(writer);
115            finish(buffer)
116        }
117    }
118}
119
120/// arrow-json's options with no encoder factory, which [`NonFiniteFloats`]
121/// builds its fallback encoders with: the same options would hand the array
122/// straight back to the factory.
123static PLAIN: LazyLock<EncoderOptions> = LazyLock::new(EncoderOptions::default);
124
125/// Writes NaN, infinity and negative infinity as the JSON strings `"NaN"`,
126/// `"Infinity"` and `"-Infinity"` (#124).
127///
128/// JSON has no number for any of the three, and arrow-json writes all of
129/// them as `null`: a NaN read back as a NULL, which a query engine must not
130/// confuse. These strings are what Python's `float()` and JavaScript's
131/// `Number()` read back as the same values. Every finite value is handed to
132/// arrow-json's own encoder, so its formatting does not change. arrow-json
133/// asks the factory for every array it encodes, the values of lists,
134/// structs and dictionaries included, so a float nested anywhere is covered.
135#[derive(Debug)]
136struct NonFiniteFloats;
137
138impl EncoderFactory for NonFiniteFloats {
139    fn make_default_encoder<'a>(
140        &self,
141        field: &'a FieldRef,
142        array: &'a dyn Array,
143        _options: &'a EncoderOptions,
144    ) -> Result<Option<NullableEncoder<'a>>, ArrowError> {
145        let encoder: Box<dyn Encoder + 'a> = match array.data_type() {
146            DataType::Float16 => Box::new(FloatEncoder::new(
147                array.as_primitive::<Float16Type>(),
148                field,
149                |v| v.to_f64(),
150            )?),
151            DataType::Float32 => Box::new(FloatEncoder::new(
152                array.as_primitive::<Float32Type>(),
153                field,
154                f64::from,
155            )?),
156            DataType::Float64 => Box::new(FloatEncoder::new(
157                array.as_primitive::<Float64Type>(),
158                field,
159                |v| v,
160            )?),
161            _ => return Ok(None),
162        };
163        Ok(Some(NullableEncoder::new(encoder, array.nulls().cloned())))
164    }
165}
166
167/// One float array: the three non-finite values as strings, everything else
168/// through arrow-json's encoder for the same array.
169struct FloatEncoder<'a, T: ArrowPrimitiveType> {
170    array: &'a PrimitiveArray<T>,
171    to_f64: fn(T::Native) -> f64,
172    finite: NullableEncoder<'a>,
173}
174
175impl<'a, T: ArrowPrimitiveType> FloatEncoder<'a, T> {
176    fn new(
177        array: &'a PrimitiveArray<T>,
178        field: &'a FieldRef,
179        to_f64: fn(T::Native) -> f64,
180    ) -> Result<Self, ArrowError> {
181        Ok(Self {
182            array,
183            to_f64,
184            finite: make_encoder(field, array, &PLAIN)?,
185        })
186    }
187}
188
189impl<T: ArrowPrimitiveType> Encoder for FloatEncoder<'_, T> {
190    fn encode(&mut self, idx: usize, out: &mut Vec<u8>) {
191        let value = (self.to_f64)(self.array.value(idx));
192        if value.is_nan() {
193            out.extend_from_slice(b"\"NaN\"");
194        } else if value == f64::INFINITY {
195            out.extend_from_slice(b"\"Infinity\"");
196        } else if value == f64::NEG_INFINITY {
197            out.extend_from_slice(b"\"-Infinity\"");
198        } else {
199            self.finite.encode(idx, out);
200        }
201    }
202}
203
204/// Turns a writer's buffer into a newline-terminated `String`.
205fn finish(mut buffer: Vec<u8>) -> Result<String, EngineError> {
206    if !buffer.ends_with(b"\n") {
207        buffer.write_all(b"\n").map_err(EngineError::from)?;
208    }
209    String::from_utf8(buffer)
210        .map_err(|e| EngineError::execution(format!("rendered output was not UTF-8: {e}")))
211}
212
213#[cfg(test)]
214#[allow(clippy::unwrap_used, clippy::expect_used)]
215mod tests {
216    use std::sync::Arc;
217
218    use datafusion::arrow::array::{
219        ArrayRef, DictionaryArray, FixedSizeListArray, Float32Array, Float64Array, Int8Array,
220        Int64Array, StringArray, StructArray,
221    };
222    use datafusion::arrow::compute::cast;
223    use datafusion::arrow::datatypes::{DataType, Field, Int8Type, Schema};
224
225    use super::*;
226
227    fn schema() -> SchemaRef {
228        Arc::new(Schema::new(vec![
229            Field::new("k", DataType::Int64, true),
230            Field::new("name", DataType::Utf8, true),
231        ]))
232    }
233
234    fn batch() -> RecordBatch {
235        RecordBatch::try_new(
236            schema(),
237            vec![
238                Arc::new(Int64Array::from(vec![Some(1), None])),
239                Arc::new(StringArray::from(vec![Some("a,b"), None])),
240            ],
241        )
242        .unwrap()
243    }
244
245    #[test]
246    fn every_name_round_trips() {
247        for format in OutputFormat::ALL {
248            assert_eq!(format.as_str().parse::<OutputFormat>().unwrap(), format);
249        }
250        assert_eq!("  CSV ".parse::<OutputFormat>().unwrap(), OutputFormat::Csv);
251    }
252
253    #[test]
254    fn an_unknown_name_names_the_alternatives() {
255        let message = "yaml".parse::<OutputFormat>().unwrap_err().to_string();
256        assert!(message.contains("yaml"), "{message}");
257        assert!(message.contains("table, json, csv"), "{message}");
258    }
259
260    /// Every object carries every column, a NULL as `null` (#124). arrow-json
261    /// leaves a NULL's key out by default, and a script could not tell a
262    /// NULL from a column that does not exist: this row used to be `{}`.
263    #[test]
264    fn json_is_an_array_of_objects_with_every_column() {
265        let text = render(&schema(), &[batch()], OutputFormat::Json).unwrap();
266        assert_eq!(
267            text,
268            "[{\"k\":1,\"name\":\"a,b\"},{\"k\":null,\"name\":null}]\n"
269        );
270    }
271
272    /// NaN and the infinities have no JSON number, and arrow-json wrote all
273    /// three as `null`, which is how a NULL is written (#124). They are
274    /// strings now, in every float type and inside a list, while finite
275    /// values and NULLs keep their encoding.
276    #[test]
277    fn json_writes_nan_and_the_infinities_as_strings() {
278        let item = Arc::new(Field::new("item", DataType::Float32, true));
279        let schema = Arc::new(Schema::new(vec![
280            Field::new("d", DataType::Float64, true),
281            Field::new("f", DataType::Float32, true),
282            Field::new("h", DataType::Float16, true),
283            Field::new("v", DataType::FixedSizeList(Arc::clone(&item), 2), true),
284        ]));
285        let d = Float64Array::from(vec![
286            Some(f64::NAN),
287            Some(f64::INFINITY),
288            Some(f64::NEG_INFINITY),
289            Some(1.5),
290            None,
291        ]);
292        let f = Float32Array::from(vec![
293            Some(f32::NEG_INFINITY),
294            Some(f32::NAN),
295            Some(0.25),
296            Some(f32::INFINITY),
297            None,
298        ]);
299        // Arrow's cast keeps NaN and the infinities, and spares this crate a
300        // dependency on `half` for one test column.
301        let h = cast(
302            &Float32Array::from(vec![
303                Some(0.5),
304                Some(f32::NEG_INFINITY),
305                Some(f32::NAN),
306                Some(f32::INFINITY),
307                None,
308            ]),
309            &DataType::Float16,
310        )
311        .unwrap();
312        let v = FixedSizeListArray::new(
313            item,
314            2,
315            Arc::new(Float32Array::from(vec![
316                Some(f32::NAN),
317                Some(0.5),
318                None,
319                Some(f32::INFINITY),
320                Some(0.0),
321                Some(0.0),
322                Some(0.0),
323                Some(0.0),
324                Some(0.0),
325                Some(0.0),
326            ])),
327            Some(vec![true, true, true, true, false].into()),
328        );
329        let batch = RecordBatch::try_new(
330            Arc::clone(&schema),
331            vec![Arc::new(d), Arc::new(f), h, Arc::new(v)],
332        )
333        .unwrap();
334        let text = render(&schema, &[batch], OutputFormat::Json).unwrap();
335        assert_eq!(
336            text,
337            concat!(
338                "[{\"d\":\"NaN\",\"f\":\"-Infinity\",\"h\":0.5,\"v\":[\"NaN\",0.5]},",
339                "{\"d\":\"Infinity\",\"f\":\"NaN\",\"h\":\"-Infinity\",\"v\":[null,\"Infinity\"]},",
340                "{\"d\":\"-Infinity\",\"f\":0.25,\"h\":\"NaN\",\"v\":[0.0,0.0]},",
341                "{\"d\":1.5,\"f\":\"Infinity\",\"h\":\"Infinity\",\"v\":[0.0,0.0]},",
342                "{\"d\":null,\"f\":null,\"h\":null,\"v\":null}]\n",
343            )
344        );
345    }
346
347    /// arrow-json builds the encoder of a struct's fields and of a
348    /// dictionary's values through the same factory, so a float there is
349    /// written the same way, and a NULL field of a struct keeps its key.
350    #[test]
351    fn json_reaches_floats_inside_structs_and_dictionaries() {
352        let x = Arc::new(Float64Array::from(vec![Some(f64::NAN), None])) as ArrayRef;
353        let s = StructArray::from(vec![(
354            Arc::new(Field::new("x", DataType::Float64, true)),
355            x,
356        )]);
357        let e = DictionaryArray::<Int8Type>::try_new(
358            Int8Array::from(vec![1, 0]),
359            Arc::new(Float64Array::from(vec![f64::NEG_INFINITY, 2.5])),
360        )
361        .unwrap();
362        let schema = Arc::new(Schema::new(vec![
363            Field::new("s", s.data_type().clone(), true),
364            Field::new("e", e.data_type().clone(), true),
365        ]));
366        let batch =
367            RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(s), Arc::new(e)]).unwrap();
368        let text = render(&schema, &[batch], OutputFormat::Json).unwrap();
369        assert_eq!(
370            text,
371            "[{\"s\":{\"x\":\"NaN\"},\"e\":2.5},{\"s\":{\"x\":null},\"e\":\"-Infinity\"}]\n"
372        );
373    }
374
375    #[test]
376    fn csv_quotes_an_embedded_comma_and_keeps_the_header() {
377        let text = render(&schema(), &[batch()], OutputFormat::Csv).unwrap();
378        assert_eq!(text, "k,name\n1,\"a,b\"\n,\n");
379    }
380
381    /// An empty result still says what the columns were: a script reading the
382    /// CSV can tell "no rows" from "the query failed".
383    #[test]
384    fn an_empty_result_still_has_a_csv_header() {
385        assert_eq!(
386            render(&schema(), &[], OutputFormat::Csv).unwrap(),
387            "k,name\n"
388        );
389        assert_eq!(render(&schema(), &[], OutputFormat::Json).unwrap(), "[]\n");
390    }
391
392    #[test]
393    fn the_table_form_is_the_default() {
394        assert_eq!(OutputFormat::default(), OutputFormat::Table);
395        let text = render(&schema(), &[batch()], OutputFormat::Table).unwrap();
396        assert!(text.contains("| k "), "{text}");
397        assert!(text.ends_with("\n"), "{text}");
398    }
399}