Skip to main content

Dataset

Struct Dataset 

Source
pub struct Dataset {
    pub inputs: Vec<Vec<f32>>,
    pub targets: Vec<Vec<f32>>,
}

Fields§

§inputs: Vec<Vec<f32>>§targets: Vec<Vec<f32>>

Implementations§

Source§

impl Dataset

Source

pub fn new(inputs: Vec<Vec<f32>>, targets: Vec<Vec<f32>>) -> Self

Examples found in repository?
examples/xor.rs (lines 4-12)
3fn main() -> Result<(), Box<dyn std::error::Error>> {
4    let dataset = Dataset::new(
5        vec![
6            vec![0.0, 0.0],
7            vec![0.0, 1.0],
8            vec![1.0, 0.0],
9            vec![1.0, 1.0],
10        ],
11        vec![vec![0.0], vec![1.0], vec![1.0], vec![0.0]],
12    );
13
14    let mut model = Network::builder()
15        .input_size(2)
16        .dense(8, Activation::Tanh)
17        .dense(1, Activation::Sigmoid)
18        .loss(Loss::BinaryCrossEntropy)
19        .optimizer(Optimizer::adam(0.05))
20        .build();
21
22    model.fit(
23        &dataset,
24        TrainConfig {
25            epochs: 2_000,
26            batch_size: 4,
27            shuffle: true,
28            seed: Some(11),
29        },
30    )?;
31
32    for input in &dataset.inputs {
33        println!("{input:?} -> {:.4}", model.predict(input)?[0]);
34    }
35
36    Ok(())
37}
More examples
Hide additional examples
examples/save_load.rs (lines 4-7)
3fn main() -> Result<(), Box<dyn std::error::Error>> {
4    let dataset = Dataset::new(
5        vec![vec![0.0], vec![0.5], vec![1.0]],
6        vec![vec![0.0], vec![1.0], vec![2.0]],
7    );
8
9    let mut model = Network::builder()
10        .input_size(1)
11        .dense(6, Activation::Relu)
12        .dense(1, Activation::Linear)
13        .loss(Loss::Mse)
14        .optimizer(Optimizer::adam(0.01))
15        .build();
16
17    model.fit(
18        &dataset,
19        TrainConfig {
20            epochs: 500,
21            batch_size: 3,
22            shuffle: true,
23            seed: Some(5),
24        },
25    )?;
26
27    let path = std::env::temp_dir().join("rusting_brain_example_model.json");
28    model.save_json(&path)?;
29
30    let loaded = Network::load_json(&path)?;
31    println!("saved to {}", path.display());
32    println!("[0.25] -> {:.4}", loaded.predict(&[0.25])?[0]);
33
34    // The same weights as an ONNX graph, for a runtime that is not this one.
35    // Run it with `cargo run --example onnx_inference --features onnx --
36    // <path> 1,1 0.25` and the number printed above comes back.
37    let onnx = std::env::temp_dir().join("rusting_brain_example_model.onnx");
38    model.save_onnx(&onnx)?;
39    println!("exported to {}", onnx.display());
40
41    Ok(())
42}
examples/classification.rs (lines 4-21)
3fn main() -> Result<(), Box<dyn std::error::Error>> {
4    let dataset = Dataset::new(
5        vec![
6            vec![1.0, 0.0],
7            vec![0.9, 0.1],
8            vec![0.0, 1.0],
9            vec![0.1, 0.9],
10            vec![0.8, 0.8],
11            vec![0.7, 0.9],
12        ],
13        vec![
14            vec![1.0, 0.0, 0.0],
15            vec![1.0, 0.0, 0.0],
16            vec![0.0, 1.0, 0.0],
17            vec![0.0, 1.0, 0.0],
18            vec![0.0, 0.0, 1.0],
19            vec![0.0, 0.0, 1.0],
20        ],
21    );
22
23    let mut model = Network::builder()
24        .input_size(2)
25        .dense(10, Activation::Relu)
26        .dense(3, Activation::Softmax)
27        .loss(Loss::CrossEntropy)
28        .optimizer(Optimizer::adam(0.03))
29        .build();
30
31    model.fit(
32        &dataset,
33        TrainConfig {
34            epochs: 1_000,
35            batch_size: 6,
36            shuffle: true,
37            seed: Some(19),
38        },
39    )?;
40
41    for input in &dataset.inputs {
42        println!("{input:?} -> {:?}", model.predict(input)?);
43    }
44
45    // Loss is what the optimizer minimized; accuracy is what the result is
46    // reported in. Six samples and no held-out split, so this only says the
47    // model fitted what it was shown.
48    println!("\naccuracy {:.1}%", 100.0 * model.accuracy(&dataset)?);
49
50    Ok(())
51}
examples/regression.rs (line 16)
3fn main() -> Result<(), Box<dyn std::error::Error>> {
4    let mut inputs = Vec::new();
5    let mut targets = Vec::new();
6
7    for i in 0..20 {
8        for j in 0..20 {
9            let x0 = (i as f32) / 20.0;
10            let x1 = (j as f32) / 20.0;
11            inputs.push(vec![x0, x1]);
12            targets.push(vec![x0 + x1]);
13        }
14    }
15
16    let dataset = Dataset::new(inputs, targets);
17    let mut model = Network::builder()
18        .input_size(2)
19        .dense(8, Activation::Relu)
20        .dense(1, Activation::Linear)
21        .loss(Loss::Mse)
22        .optimizer(Optimizer::adam(0.01))
23        .build();
24
25    let initial = model.evaluate_loss(&dataset)?;
26    println!("initial loss: {initial:.6}");
27
28    // `fit_with` instead of `fit`: five hundred epochs are otherwise five
29    // hundred epochs of silence, and this problem is solved long before the
30    // last one. Returning `false` stops the loop, and the history holds only
31    // the epochs that ran.
32    let history = model.fit_with(
33        &dataset,
34        TrainConfig {
35            epochs: 500,
36            batch_size: 16,
37            shuffle: true,
38            seed: Some(3),
39        },
40        |epoch, loss| {
41            if epoch % 100 == 0 {
42                println!("  epoch {epoch:>3}  loss {loss:.6}");
43            }
44            loss > 1e-5
45        },
46    )?;
47    let final_loss = model.evaluate_loss(&dataset)?;
48
49    println!("epochs run:   {}", history.losses.len());
50    println!("final loss:   {final_loss:.6}");
51    println!("[0.25, 0.50] -> {:.4}", model.predict(&[0.25, 0.50])?[0]);
52
53    Ok(())
54}
Source

pub fn from_csv( path: impl AsRef<Path>, target_columns: usize, ) -> Result<Self, NetworkError>

Reads a comma-separated file in which the last target_columns columns are the target and every column before them is an input.

A first row that does not parse as numbers is taken to be a header and skipped. Quoted fields are understood, including "" for a literal quote.

// sepal_length,sepal_width,petal_length,is_setosa
let dataset = Dataset::from_csv("iris.csv", 1)?;
Source

pub fn from_csv_labeled( path: impl AsRef<Path>, ) -> Result<(Self, Vec<String>), NetworkError>

Reads a comma-separated file whose last column is a class name rather than a number.

This is what a classification dataset off the internet actually looks like — 5.1,3.5,1.4,0.2,setosa — and Dataset::from_csv rejects it, because the last column does not parse as a float. The distinct labels sorted become the classes, the target is a one-hot row over them, and the names come back so a prediction can be named.

let (dataset, classes) = Dataset::from_csv_labeled(&path)?;
assert_eq!(classes, ["setosa", "versicolor"]);
assert_eq!(dataset.targets[0], vec![1.0, 0.0]);
Source

pub fn from_jsonl( path: impl AsRef<Path>, input_field: &str, target_field: &str, ) -> Result<Self, NetworkError>

Reads two NumPy .npy files, one of inputs and one of targets.

This is the file numpy.save writes, which is how data leaves PyTorch, TensorFlow and scikit-learn without going through a CSV twenty times its size. A 2-D array is one row per sample; a 1-D array is one value per sample, which is what a column of labels looks like. float32, float64, int32, int64 and uint8 all arrive as f32.

// numpy.save("x.npy", x); numpy.save("y.npy", y)
let mut dataset = Dataset::from_npy("x.npy", "y.npy")?;
dataset.one_hot_targets(10)?;      // y held class indices, not one-hot rows

Reads a whole JSONL file into memory, one row per line.

Each line is an object holding input_field and target_field, each one an array of numbers or a single number. JsonlStream reads the same format a batch at a time, for a file that does not fit in memory.

let dataset = Dataset::from_jsonl("corpus.jsonl", "pixels", "label")?;
Source

pub fn from_npy( inputs: impl AsRef<Path>, targets: impl AsRef<Path>, ) -> Result<Self, NetworkError>

Source

pub fn from_idx( images: impl AsRef<Path>, labels: impl AsRef<Path>, ) -> Result<Self, NetworkError>

Reads a pair of IDX files, the format MNIST and its many descendants ship in.

images is the idx3-ubyte file and labels the idx1-ubyte one. An image of any shape is flattened into one input row, and u8 pixels are scaled to 0.0..=1.0; labels are left as the class indices they are, so one_hot_targets is the next call.

The files the archives ship are gzipped and this reads the plain ones: gunzip train-images-idx3-ubyte.gz first.

let mut train = Dataset::from_idx("train-images-idx3-ubyte", "train-labels-idx1-ubyte")?;
train.one_hot_targets(10)?;
Source

pub fn one_hot_targets(&mut self, classes: usize) -> Result<(), NetworkError>

Turns a single target column of class indices into one-hot rows over classes classes.

Labels arrive as indices far more often than as one-hot rows — from .npy, from a CSV of numbers — and a network with a softmax output needs the row.

Source

pub fn len(&self) -> usize

Source

pub fn is_empty(&self) -> bool

Source

pub fn shuffle(&mut self, seed: Option<u64>)

Source

pub fn split_stratified(&self, train_ratio: f32) -> (Self, Self)

split, keeping each class’s share of the rows on both sides.

A plain split of an imbalanced set can leave a rare class out of the test half entirely, and the accuracy it reports then says nothing about that class. Classes are read off the targets: the largest entry of a one-hot row, or the value itself when the target is a single column.

Row order inside a class is kept, so shuffle first — the split is otherwise the first rows of each class, which is whatever order the loader produced.

let mut dataset = Dataset::new(vec![vec![0.0]; 10], vec![vec![1.0, 0.0]; 10]);
dataset.targets[9] = vec![0.0, 1.0];             // one row of a rare class
let (train, test) = dataset.split_stratified(0.8);
assert_eq!((train.len(), test.len()), (8, 2));
Source

pub fn split(&self, train_ratio: f32) -> (Self, Self)

Source

pub fn standardize(&mut self) -> Standardizer

Centers every input column on zero and scales it to unit deviation, returning the statistics it used.

A network fed raw columns — an age beside an income — spends its first epochs undoing the scale difference, and with a saturating activation it may never recover. Fit on the training split and apply those statistics to the test split: fitting again on the test split leaks it.

let mut train = Dataset::new(vec![vec![1.0, 900.0], vec![3.0, 1100.0]], vec![vec![0.0]; 2]);
let mut test = Dataset::new(vec![vec![2.0, 1000.0]], vec![vec![1.0]]);

let statistics = train.standardize();
statistics.apply(&mut test);
assert_eq!(test.inputs[0], vec![0.0, 0.0]);
Source

pub fn flip_horizontal(&mut self, width: usize) -> Result<(), NetworkError>

Doubles the dataset with a left-right mirror of every input row.

A flipped photograph of a cat is still a photograph of a cat, so the mirrored copy keeps its target and the model sees twice the data for the price of one decode. Rows are the flat layout from_image_folder produces — plane after plane, each plane row-major of width pixels — so an RGB image flips correctly with the same width as a grayscale one.

Apply it to the training split only. Augmenting before split puts an image and its mirror on opposite sides of the split and the test accuracy stops meaning anything.

let mut dataset = Dataset::new(vec![vec![1.0, 2.0, 3.0, 4.0]], vec![vec![1.0]]);
dataset.flip_horizontal(2)?;                      // two 2x1 rows
assert_eq!(dataset.inputs[1], vec![2.0, 1.0, 4.0, 3.0]);
assert_eq!(dataset.targets[1], vec![1.0]);

This doubles the dataset once, up front. Random crops and rotations, which want a fresh draw every epoch, are Augment.

Source

pub fn stream(&mut self, config: &TrainConfig) -> DatasetStream<'_>

Reads this dataset as a BatchSource, which is what Network::fit trains from.

The stream takes the batch size, the shuffling and the seed from config, and shuffles the dataset in place at the start of every epoch, which is why it borrows mutably.

let mut dataset = Dataset::new(vec![vec![0.0]; 5], vec![vec![1.0]; 5]);
let config = TrainConfig { batch_size: 2, shuffle: false, ..Default::default() };
let mut stream = dataset.stream(&config);

stream.start_epoch(0)?;
let mut sizes = Vec::new();
while let Some(batch) = stream.next_batch()? {
    sizes.push(batch.inputs.len());
}
assert_eq!(sizes, [2, 2, 1]);
Source

pub fn batches(&self, batch_size: usize) -> Vec<DatasetBatch<'_>>

Trait Implementations§

Source§

impl Clone for Dataset

Source§

fn clone(&self) -> Self

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for Dataset

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more
Source§

impl PartialEq for Dataset

Source§

fn eq(&self, other: &Self) -> bool

Equality operator ==. Read more
1.0.0 (const: unstable) · Source§

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=. Read more
Source§

impl StructuralPartialEq for Dataset

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

impl<T> Pointable for T

Source§

const ALIGN: usize

The alignment of pointer.
Source§

type Init = T

The type for initializers.
Source§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
Source§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
Source§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
Source§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V