Struct SwapVec 

pub struct SwapVec<T>
where
    for<'a> T: Serialize + Deserialize<'a>,
{ /* private fields */ }

An only growing array type which swaps to disk, based on it’s initial configuration.

Create a mutable instance, and then pass iterators or elements to grow it.

let mut bigvec = swapvec::SwapVec::default();
let iterator = (0..9);
bigvec.consume(iterator);
bigvec.push(99);
let new_iterator = bigvec.into_iter();

Implementations

impl<T> SwapVec<T>

where for<'a> T: Serialize + Deserialize<'a> + Clone,

pub fn with_config(config: SwapVecConfig) -> Self

Intialize with non-default configuration.

Examples found in repository

examples/demo.rs (line 13)

fn main() {
    let element_count = DATA_MB / 8;
    let big_iterator = 0..element_count * 1024 * 1024;

    let config = swapvec::SwapVecConfig {
        batch_size: 8 * 1024,
        ..SwapVecConfig::default()
    };
    let mut swapvec: SwapVec<_> = SwapVec::with_config(config);
    swapvec.consume(big_iterator.into_iter()).unwrap();

    println!("Data size: {}MB", DATA_MB);
    println!("Done. Batches written: {}", swapvec.batches_written());
    println!(
        "Filesize: {}MB",
        swapvec
            .file_size()
            .map(|x| x as f32 / 1024. / 1024.)
            .unwrap_or(0.)
    );
    println!("Read back");

    let read_back: Vec<_> = swapvec.into_iter().map(|x| x.unwrap()).collect();

    println!("Elements read back: {}", read_back.len());
}

pub fn consume( &mut self, it: impl Iterator<Item = T>, ) -> Result<(), SwapVecError>

Give away an entire iterator for consumption.
Might return an error, due to possibly triggered batch flush (IO).

Examples found in repository

examples/demo.rs (line 14)

fn main() {
    let element_count = DATA_MB / 8;
    let big_iterator = 0..element_count * 1024 * 1024;

    let config = swapvec::SwapVecConfig {
        batch_size: 8 * 1024,
        ..SwapVecConfig::default()
    };
    let mut swapvec: SwapVec<_> = SwapVec::with_config(config);
    swapvec.consume(big_iterator.into_iter()).unwrap();

    println!("Data size: {}MB", DATA_MB);
    println!("Done. Batches written: {}", swapvec.batches_written());
    println!(
        "Filesize: {}MB",
        swapvec
            .file_size()
            .map(|x| x as f32 / 1024. / 1024.)
            .unwrap_or(0.)
    );
    println!("Read back");

    let read_back: Vec<_> = swapvec.into_iter().map(|x| x.unwrap()).collect();

    println!("Elements read back: {}", read_back.len());
}

pub fn push(&mut self, element: T) -> Result<(), SwapVecError>

Push a single element. Might return an error, due to possibly triggered batch flush (IO). Will write at most one batch per insert. If swap_after is bigger than batch_size and a file is created, every insert will write one batch to disk, until the elements in memory have a count smaller than or equal to batch size.

pub fn written_to_file(&self) -> bool

Check if enough items have been pushed so that the temporary file has been created.
Will be false if element count is below swap_after and below batch_size

pub fn file_size(&self) -> Option<usize>

Get the file size in bytes of the temporary file. Might do IO and therefore could return some Result.

Examples found in repository

examples/demo.rs (line 21)

fn main() {
    let element_count = DATA_MB / 8;
    let big_iterator = 0..element_count * 1024 * 1024;

    let config = swapvec::SwapVecConfig {
        batch_size: 8 * 1024,
        ..SwapVecConfig::default()
    };
    let mut swapvec: SwapVec<_> = SwapVec::with_config(config);
    swapvec.consume(big_iterator.into_iter()).unwrap();

    println!("Data size: {}MB", DATA_MB);
    println!("Done. Batches written: {}", swapvec.batches_written());
    println!(
        "Filesize: {}MB",
        swapvec
            .file_size()
            .map(|x| x as f32 / 1024. / 1024.)
            .unwrap_or(0.)
    );
    println!("Read back");

    let read_back: Vec<_> = swapvec.into_iter().map(|x| x.unwrap()).collect();

    println!("Elements read back: {}", read_back.len());
}

pub fn batches_written(&self) -> usize

Basically int(elements pushed / batch size)

Examples found in repository

examples/demo.rs (line 17)

fn main() {
    let element_count = DATA_MB / 8;
    let big_iterator = 0..element_count * 1024 * 1024;

    let config = swapvec::SwapVecConfig {
        batch_size: 8 * 1024,
        ..SwapVecConfig::default()
    };
    let mut swapvec: SwapVec<_> = SwapVec::with_config(config);
    swapvec.consume(big_iterator.into_iter()).unwrap();

    println!("Data size: {}MB", DATA_MB);
    println!("Done. Batches written: {}", swapvec.batches_written());
    println!(
        "Filesize: {}MB",
        swapvec
            .file_size()
            .map(|x| x as f32 / 1024. / 1024.)
            .unwrap_or(0.)
    );
    println!("Read back");

    let read_back: Vec<_> = swapvec.into_iter().map(|x| x.unwrap()).collect();

    println!("Elements read back: {}", read_back.len());
}

Trait Implementations

impl<T: Serialize + for<'a> Deserialize<'a>> Debug for SwapVec<T>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<T: Serialize + for<'a> Deserialize<'a>> Default for SwapVec<T>

fn default() -> Self

Returns the “default value” for a type.

impl<T: Serialize + for<'a> Deserialize<'a> + Clone> IntoIterator for SwapVec<T>

type Item = Result<T, SwapVecError>

The type of the elements being iterated over.

type IntoIter = SwapVecIter<T>

Which kind of iterator are we turning this into?

fn into_iter(self) -> Self::IntoIter

Creates an iterator from a value.