use std::io::Write;
use arrow::csv::WriterBuilder as CsvWriterBuilder;
use arrow::datatypes::SchemaRef;
use arrow::error::ArrowError;
use arrow::ipc::writer::StreamWriter;
use arrow::json::{ArrayWriter, LineDelimitedWriter};
use arrow::record_batch::RecordBatch;
use arrow::util::display::FormatOptions;
use arrow::util::pretty::pretty_format_batches_with_options;
use datafusion::error::{DataFusionError, Result};
use datafusion::physical_plan::SendableRecordBatchStream;
use futures::StreamExt;
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
pub enum OutputFormat {
Table,
Csv,
Tsv,
Json,
Ndjson,
Arrow,
}
pub async fn write_stream<W: Write>(
format: OutputFormat,
mut stream: SendableRecordBatchStream,
out: &mut W,
options: &FormatOptions<'_>,
) -> Result<()> {
let schema = stream.schema();
match format {
OutputFormat::Table => {
let mut batches = Vec::new();
while let Some(batch) = stream.next().await {
let batch = batch?;
if batch.num_rows() > 0 {
batches.push(batch);
}
}
if batches.is_empty() {
batches.push(RecordBatch::new_empty(schema));
}
writeln!(
out,
"{}",
pretty_format_batches_with_options(&batches, options)?
)?;
}
OutputFormat::Csv | OutputFormat::Tsv => {
let delimiter = if format == OutputFormat::Csv {
b','
} else {
b'\t'
};
let mut sink = BrokenPipeAware::new(&mut *out);
let mut writer = CsvWriterBuilder::new()
.with_header(true)
.with_delimiter(delimiter)
.build(&mut sink);
let mut wrote = false;
let mut written = Ok(());
while let Some(batch) = stream.next().await {
written = writer.write(&batch?);
if written.is_err() {
break;
}
wrote = true;
}
if written.is_ok() && !wrote {
written = writer.write(&RecordBatch::new_empty(schema));
}
drop(writer);
if sink.broken {
return Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe).into());
}
written?;
}
OutputFormat::Json => {
let mut writer = ArrayWriter::new(&mut *out);
while let Some(batch) = stream.next().await {
writer.write(&batch?)?;
}
writer.finish()?;
drop(writer);
writeln!(out)?;
}
OutputFormat::Ndjson => {
let mut writer = LineDelimitedWriter::new(&mut *out);
while let Some(batch) = stream.next().await {
writer.write(&batch?)?;
}
writer.finish()?;
}
OutputFormat::Arrow => {
let mut writer = StreamWriter::try_new(&mut *out, &schema)?;
while let Some(batch) = stream.next().await {
writer.write(&batch?)?;
}
writer.finish()?;
}
}
out.flush()?;
Ok(())
}
struct BrokenPipeAware<W> {
inner: W,
broken: bool,
}
impl<W: Write> BrokenPipeAware<W> {
fn new(inner: W) -> Self {
Self {
inner,
broken: false,
}
}
fn note<T>(&mut self, result: std::io::Result<T>) -> std::io::Result<T> {
if let Err(e) = &result {
if e.kind() == std::io::ErrorKind::BrokenPipe {
self.broken = true;
}
}
result
}
}
impl<W: Write> Write for BrokenPipeAware<W> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let result = self.inner.write(buf);
self.note(result)
}
fn flush(&mut self) -> std::io::Result<()> {
let result = self.inner.flush();
self.note(result)
}
}
pub fn has_columns(schema: &SchemaRef) -> bool {
!schema.fields().is_empty()
}
pub fn is_broken_pipe(err: &DataFusionError) -> bool {
match err {
DataFusionError::IoError(e) => e.kind() == std::io::ErrorKind::BrokenPipe,
DataFusionError::ArrowError(e, _) => {
matches!(e.as_ref(), ArrowError::IoError(_, e) if e.kind() == std::io::ErrorKind::BrokenPipe)
}
_ => false,
}
}