Skip to main content

rs_df2ipc_stream/
lib.rs

1use std::io;
2use std::sync::Arc;
3
4use io::BufWriter;
5use io::Write;
6
7use futures::StreamExt;
8use futures::TryStreamExt;
9
10use arrow::datatypes::Schema;
11use arrow::ipc::writer::StreamWriter;
12use arrow::record_batch::RecordBatch;
13
14use datafusion::dataframe::DataFrame;
15use datafusion::execution::SendableRecordBatchStream;
16
17pub struct IpcStreamWriter<W: Write>(pub StreamWriter<BufWriter<W>>);
18
19impl<W> IpcStreamWriter<W>
20where
21    W: Write,
22{
23    pub async fn write_batch_stream(
24        self,
25        strm: SendableRecordBatchStream,
26    ) -> Result<Self, io::Error> {
27        let mut me: Self = strm
28            .map(|r| r.map_err(io::Error::other))
29            .try_fold(self, |mut state, next| async move {
30                let rbat: &RecordBatch = &next;
31                state.0.write(rbat).map_err(io::Error::other)?;
32                Ok(state)
33            })
34            .await?;
35        me.0.flush().map_err(io::Error::other)?;
36        Ok(me)
37    }
38
39    pub async fn df2writer(self, df: DataFrame) -> Result<Self, io::Error> {
40        let strm = df.execute_stream().await?;
41        self.write_batch_stream(strm).await
42    }
43}
44
45pub async fn df2stdout(df: DataFrame) -> Result<(), io::Error> {
46    let mut o = io::stdout().lock();
47    let sch: Arc<Schema> = df.schema().inner().clone();
48    let swtr: StreamWriter<_> =
49        StreamWriter::try_new_buffered(&mut o, &sch).map_err(io::Error::other)?;
50    let mut iwtr: IpcStreamWriter<_> = IpcStreamWriter(swtr).df2writer(df).await?;
51    iwtr.0.finish().map_err(io::Error::other)?;
52    drop(iwtr);
53    o.flush()?;
54    Ok(())
55}