dataz 0.3.0

High-throughput generative datasets
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

// Copyright 2022 Daniel Harrison. All Rights Reserved.

//! High-throughput computational dataset synthesis

#![warn(missing_docs, missing_debug_implementations)]

use crate::col::{Col, Data};

pub mod col;
pub mod kvtd;

#[cfg(feature = "serde")]
pub mod serde;

/// A named dataset made up of one or more [Table]s.
pub trait Set {
    /// Configuration necessary for construct this dataset.
    type Config;

    /// Construct an instance of this dataset with the given configuration.
    fn init(config: Self::Config) -> Self;

    /// The tables making up this dataset.
    fn tables(&self) -> Vec<&dyn DynTable>;
}

/// A type-erased [Table].
pub trait DynTable {
    /// A human-readable name for this table.
    fn name(&self) -> &'static str;

    /// The number of batches of data in this table.
    fn num_batches(&self) -> usize;
}

/// A named set of data with a uniform schema.
///
/// A table is generated in batches, each of which contains one or more rows.
/// For each parallelization, each batch can be generated purely as a function
/// of its index and dataset configuration. For vectorization, a batch is
/// internally arranged in columns.
pub trait Table: DynTable {
    /// The type data generated by this table.
    type Data: Data;

    /// Generates the requested batch's data.
    ///
    /// This clears the given batch and reuses allocations when possible. If the
    /// requested index is out of bounds, an empty batch is generated.
    fn gen_batch<C: Col<Self::Data>>(&self, idx: usize, batch: &mut C);
}