io_ipc
and io_ipc_write_async
only.Expand description
Sink that writes array chunks
as an IPC file.
The file header is automatically written before writing the first chunk, and the file footer is automatically written when the sink is closed.
Examples
use std::sync::Arc;
use futures::{SinkExt, TryStreamExt, io::Cursor};
use arrow2::array::{Array, Int32Array};
use arrow2::datatypes::{DataType, Field, Schema};
use arrow2::chunk::Chunk;
use arrow2::io::ipc::write::file_async::FileSink;
use arrow2::io::ipc::read::file_async::{read_file_metadata_async, FileStream};
let schema = Schema::from(vec![
Field::new("values", DataType::Int32, true),
]);
let mut buffer = Cursor::new(vec![]);
let mut sink = FileSink::new(
&mut buffer,
&schema,
None,
Default::default(),
);
// Write chunks to file
for i in 0..3 {
let values = Int32Array::from(&[Some(i), None]);
let chunk = Chunk::new(vec![Arc::new(values) as Arc<dyn Array>]);
sink.feed(chunk.into()).await?;
}
sink.close().await?;
drop(sink);
// Read chunks from file
buffer.set_position(0);
let metadata = read_file_metadata_async(&mut buffer).await?;
let mut stream = FileStream::new(buffer, metadata, None);
let chunks = stream.try_collect::<Vec<_>>().await?;
Implementations
Trait Implementations
sourceimpl<'a, W> Sink<Record<'_>> for FileSink<'a, W> where
W: AsyncWrite + Unpin + Send + 'a,
impl<'a, W> Sink<Record<'_>> for FileSink<'a, W> where
W: AsyncWrite + Unpin + Send + 'a,
type Error = ArrowError
type Error = ArrowError
The type of value produced by the sink when an error occurs.
sourcefn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>>
fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<()>>
Attempts to prepare the Sink
to receive a value. Read more
sourcefn start_send(self: Pin<&mut Self>, item: Record<'_>) -> Result<()>
fn start_send(self: Pin<&mut Self>, item: Record<'_>) -> Result<()>
Begin the process of sending a value to the sink.
Each call to this function must be preceded by a successful call to
poll_ready
which returned Poll::Ready(Ok(()))
. Read more
Auto Trait Implementations
impl<'a, W> !RefUnwindSafe for FileSink<'a, W>
impl<'a, W> Send for FileSink<'a, W>
impl<'a, W> !Sync for FileSink<'a, W>
impl<'a, W> Unpin for FileSink<'a, W>
impl<'a, W> !UnwindSafe for FileSink<'a, W>
Blanket Implementations
sourceimpl<T> BorrowMut<T> for T where
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
const: unstable · sourcefn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
impl<T, Item> SinkExt<Item> for T where
T: Sink<Item> + ?Sized,
impl<T, Item> SinkExt<Item> for T where
T: Sink<Item> + ?Sized,
fn with<U, Fut, F, E>(self, f: F) -> With<Self, Item, U, Fut, F> where
F: FnMut(U) -> Fut,
Fut: Future<Output = Result<Item, E>>,
E: From<Self::Error>,
fn with<U, Fut, F, E>(self, f: F) -> With<Self, Item, U, Fut, F> where
F: FnMut(U) -> Fut,
Fut: Future<Output = Result<Item, E>>,
E: From<Self::Error>,
Composes a function in front of the sink. Read more
fn with_flat_map<U, St, F>(self, f: F) -> WithFlatMap<Self, Item, U, St, F> where
F: FnMut(U) -> St,
St: Stream<Item = Result<Item, Self::Error>>,
fn with_flat_map<U, St, F>(self, f: F) -> WithFlatMap<Self, Item, U, St, F> where
F: FnMut(U) -> St,
St: Stream<Item = Result<Item, Self::Error>>,
Composes a function in front of the sink. Read more
fn sink_map_err<E, F>(self, f: F) -> SinkMapErr<Self, F> where
F: FnOnce(Self::Error) -> E,
fn sink_map_err<E, F>(self, f: F) -> SinkMapErr<Self, F> where
F: FnOnce(Self::Error) -> E,
Transforms the error returned by the sink.
fn sink_err_into<E>(self) -> SinkErrInto<Self, Item, E> where
Self::Error: Into<E>,
fn sink_err_into<E>(self) -> SinkErrInto<Self, Item, E> where
Self::Error: Into<E>,
Map this sink’s error to a different error type using the Into
trait. Read more
fn buffer(self, capacity: usize) -> Buffer<Self, Item>
fn buffer(self, capacity: usize) -> Buffer<Self, Item>
Adds a fixed-size buffer to the current sink. Read more
fn fanout<Si>(self, other: Si) -> Fanout<Self, Si> where
Item: Clone,
Si: Sink<Item, Error = Self::Error>,
fn fanout<Si>(self, other: Si) -> Fanout<Self, Si> where
Item: Clone,
Si: Sink<Item, Error = Self::Error>,
Fanout items to multiple sinks. Read more
fn flush(&mut self) -> Flush<'_, Self, Item> where
Self: Unpin,
fn flush(&mut self) -> Flush<'_, Self, Item> where
Self: Unpin,
Flush the sink, processing all pending items. Read more
fn send(&mut self, item: Item) -> Send<'_, Self, Item> where
Self: Unpin,
fn send(&mut self, item: Item) -> Send<'_, Self, Item> where
Self: Unpin,
A future that completes after the given item has been fully processed into the sink, including flushing. Read more
fn feed(&mut self, item: Item) -> Feed<'_, Self, Item> where
Self: Unpin,
fn feed(&mut self, item: Item) -> Feed<'_, Self, Item> where
Self: Unpin,
A future that completes after the given item has been received by the sink. Read more
fn send_all<St>(&'a mut self, stream: &'a mut St) -> SendAll<'a, Self, St> where
St: TryStream<Ok = Item, Error = Self::Error> + Stream + Unpin + ?Sized,
Self: Unpin,
fn send_all<St>(&'a mut self, stream: &'a mut St) -> SendAll<'a, Self, St> where
St: TryStream<Ok = Item, Error = Self::Error> + Stream + Unpin + ?Sized,
Self: Unpin,
A future that completes after the given stream has been fully processed into the sink, including flushing. Read more
fn left_sink<Si2>(self) -> Either<Self, Si2> where
Si2: Sink<Item, Error = Self::Error>,
fn left_sink<Si2>(self) -> Either<Self, Si2> where
Si2: Sink<Item, Error = Self::Error>,
Wrap this sink in an Either
sink, making it the left-hand variant
of that Either
. Read more
fn right_sink<Si1>(self) -> Either<Si1, Self> where
Si1: Sink<Item, Error = Self::Error>,
fn right_sink<Si1>(self) -> Either<Si1, Self> where
Si1: Sink<Item, Error = Self::Error>,
Wrap this stream in an Either
stream, making it the right-hand variant
of that Either
. Read more