Skip to main content

Crate embeddenator_io

Crate embeddenator_io 

Source
Expand description

§embeddenator-io

I/O utilities and serialization for Embeddenator.

Extracted from embeddenator core as part of Phase 2A component decomposition.

§Features

  • Serialization: Bincode and JSON support for efficient data encoding
  • Buffering: Optimized buffered I/O with configurable buffer sizes
  • Streaming: Memory-efficient streaming I/O for large files
  • Envelope Format: Compressed binary envelope format with multiple codecs
  • Async Support: Optional async I/O with tokio (enable async feature)

§Examples

§Serialization

use embeddenator_io::{to_bincode, from_bincode};
use serde::{Serialize, Deserialize};

#[derive(Serialize, Deserialize, PartialEq, Debug)]
struct Data { value: u32 }

let data = Data { value: 42 };
let bytes = to_bincode(&data).unwrap();
let decoded: Data = from_bincode(&bytes).unwrap();
assert_eq!(data, decoded);

§Streaming

use embeddenator_io::stream_read_file;

let mut total = 0;
stream_read_file("large_file.bin", |chunk| {
    total += chunk.len();
    Ok(())
}).unwrap();

§Envelope Format

use embeddenator_io::{PayloadKind, BinaryWriteOptions, wrap_or_legacy, unwrap_auto};

let data = b"Hello, world!";
let opts = BinaryWriteOptions::default();
let wrapped = wrap_or_legacy(PayloadKind::EngramBincode, opts, data).unwrap();
let unwrapped = unwrap_auto(PayloadKind::EngramBincode, &wrapped).unwrap();
assert_eq!(data, unwrapped.as_slice());

Re-exports§

pub use buffer::buffered_reader;
pub use buffer::buffered_writer;
pub use buffer::copy_buffered;
pub use buffer::read_chunks;
pub use buffer::write_chunks;
pub use buffer::ChunkStream;
pub use buffer::DEFAULT_BUFFER_SIZE;
pub use buffer::LARGE_BUFFER_SIZE;
pub use buffer::SMALL_BUFFER_SIZE;
pub use serialize::from_bincode;
pub use serialize::from_json;
pub use serialize::read_bincode_file;
pub use serialize::read_json_file;
pub use serialize::to_bincode;
pub use serialize::to_json;
pub use serialize::to_json_pretty;
pub use serialize::write_bincode_file;
pub use serialize::write_json_file;
pub use stream::stream_read_file;
pub use stream::stream_write_file;
pub use stream::StreamReader;
pub use stream::StreamWriter;
pub use io::*;

Modules§

io