Struct datafusion::common::arrow::ipc::writer::IpcDataGenerator

pub struct IpcDataGenerator {}

Handles low level details of encoding Array and Schema into the Arrow IPC Format.

Example:

// Create a record batch
let batch = RecordBatch::try_from_iter(vec![
 ("col2", Arc::new(UInt64Array::from_iter([10, 23, 33])) as _)
]).unwrap();

// Error of dictionary ids are replaced.
let error_on_replacement = true;
let options = IpcWriteOptions::default();
let mut dictionary_tracker = DictionaryTracker::new(error_on_replacement);

// encode the batch into zero or more encoded dictionaries
// and the data for the actual array.
let data_gen = IpcDataGenerator {};
let (encoded_dictionaries, encoded_message) = data_gen
  .encoded_batch(&batch, &mut dictionary_tracker, &options)
  .unwrap();

Implementations

impl IpcDataGenerator

pub fn schema_to_bytes( &self, schema: &Schema, write_options: &IpcWriteOptions, ) -> EncodedData

pub fn encoded_batch( &self, batch: &RecordBatch, dictionary_tracker: &mut DictionaryTracker, write_options: &IpcWriteOptions, ) -> Result<(Vec<EncodedData>, EncodedData), ArrowError>

Encodes a batch to a number of EncodedData items (dictionary batches + the record batch). The DictionaryTracker keeps track of dictionaries with new dict_ids (so they are only sent once) Make sure the DictionaryTracker is initialized at the start of the stream.

Trait Implementations

impl Debug for IpcDataGenerator

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Default for IpcDataGenerator

fn default() -> IpcDataGenerator

Returns the “default value” for a type.