pub struct Dataset {
pub inputs: Vec<Vec<f32>>,
pub targets: Vec<Vec<f32>>,
}Fields§
§inputs: Vec<Vec<f32>>§targets: Vec<Vec<f32>>Implementations§
Source§impl Dataset
impl Dataset
Sourcepub fn new(inputs: Vec<Vec<f32>>, targets: Vec<Vec<f32>>) -> Self
pub fn new(inputs: Vec<Vec<f32>>, targets: Vec<Vec<f32>>) -> Self
Examples found in repository?
3fn main() -> Result<(), Box<dyn std::error::Error>> {
4 let dataset = Dataset::new(
5 vec![
6 vec![0.0, 0.0],
7 vec![0.0, 1.0],
8 vec![1.0, 0.0],
9 vec![1.0, 1.0],
10 ],
11 vec![vec![0.0], vec![1.0], vec![1.0], vec![0.0]],
12 );
13
14 let mut model = Network::builder()
15 .input_size(2)
16 .dense(8, Activation::Tanh)
17 .dense(1, Activation::Sigmoid)
18 .loss(Loss::BinaryCrossEntropy)
19 .optimizer(Optimizer::adam(0.05))
20 .build();
21
22 model.fit(
23 &dataset,
24 TrainConfig {
25 epochs: 2_000,
26 batch_size: 4,
27 shuffle: true,
28 seed: Some(11),
29 },
30 )?;
31
32 for input in &dataset.inputs {
33 println!("{input:?} -> {:.4}", model.predict(input)?[0]);
34 }
35
36 Ok(())
37}More examples
3fn main() -> Result<(), Box<dyn std::error::Error>> {
4 let dataset = Dataset::new(
5 vec![vec![0.0], vec![0.5], vec![1.0]],
6 vec![vec![0.0], vec![1.0], vec![2.0]],
7 );
8
9 let mut model = Network::builder()
10 .input_size(1)
11 .dense(6, Activation::Relu)
12 .dense(1, Activation::Linear)
13 .loss(Loss::Mse)
14 .optimizer(Optimizer::adam(0.01))
15 .build();
16
17 model.fit(
18 &dataset,
19 TrainConfig {
20 epochs: 500,
21 batch_size: 3,
22 shuffle: true,
23 seed: Some(5),
24 },
25 )?;
26
27 let path = std::env::temp_dir().join("rusting_brain_example_model.json");
28 model.save_json(&path)?;
29
30 let loaded = Network::load_json(&path)?;
31 println!("saved to {}", path.display());
32 println!("[0.25] -> {:.4}", loaded.predict(&[0.25])?[0]);
33
34 // The same weights as an ONNX graph, for a runtime that is not this one.
35 // Run it with `cargo run --example onnx_inference --features onnx --
36 // <path> 1,1 0.25` and the number printed above comes back.
37 let onnx = std::env::temp_dir().join("rusting_brain_example_model.onnx");
38 model.save_onnx(&onnx)?;
39 println!("exported to {}", onnx.display());
40
41 Ok(())
42}3fn main() -> Result<(), Box<dyn std::error::Error>> {
4 let dataset = Dataset::new(
5 vec![
6 vec![1.0, 0.0],
7 vec![0.9, 0.1],
8 vec![0.0, 1.0],
9 vec![0.1, 0.9],
10 vec![0.8, 0.8],
11 vec![0.7, 0.9],
12 ],
13 vec![
14 vec![1.0, 0.0, 0.0],
15 vec![1.0, 0.0, 0.0],
16 vec![0.0, 1.0, 0.0],
17 vec![0.0, 1.0, 0.0],
18 vec![0.0, 0.0, 1.0],
19 vec![0.0, 0.0, 1.0],
20 ],
21 );
22
23 let mut model = Network::builder()
24 .input_size(2)
25 .dense(10, Activation::Relu)
26 .dense(3, Activation::Softmax)
27 .loss(Loss::CrossEntropy)
28 .optimizer(Optimizer::adam(0.03))
29 .build();
30
31 model.fit(
32 &dataset,
33 TrainConfig {
34 epochs: 1_000,
35 batch_size: 6,
36 shuffle: true,
37 seed: Some(19),
38 },
39 )?;
40
41 for input in &dataset.inputs {
42 println!("{input:?} -> {:?}", model.predict(input)?);
43 }
44
45 // Loss is what the optimizer minimized; accuracy is what the result is
46 // reported in. Six samples and no held-out split, so this only says the
47 // model fitted what it was shown.
48 println!("\naccuracy {:.1}%", 100.0 * model.accuracy(&dataset)?);
49
50 Ok(())
51}3fn main() -> Result<(), Box<dyn std::error::Error>> {
4 let mut inputs = Vec::new();
5 let mut targets = Vec::new();
6
7 for i in 0..20 {
8 for j in 0..20 {
9 let x0 = (i as f32) / 20.0;
10 let x1 = (j as f32) / 20.0;
11 inputs.push(vec![x0, x1]);
12 targets.push(vec![x0 + x1]);
13 }
14 }
15
16 let dataset = Dataset::new(inputs, targets);
17 let mut model = Network::builder()
18 .input_size(2)
19 .dense(8, Activation::Relu)
20 .dense(1, Activation::Linear)
21 .loss(Loss::Mse)
22 .optimizer(Optimizer::adam(0.01))
23 .build();
24
25 let initial = model.evaluate_loss(&dataset)?;
26 println!("initial loss: {initial:.6}");
27
28 // `fit_with` instead of `fit`: five hundred epochs are otherwise five
29 // hundred epochs of silence, and this problem is solved long before the
30 // last one. Returning `false` stops the loop, and the history holds only
31 // the epochs that ran.
32 let history = model.fit_with(
33 &dataset,
34 TrainConfig {
35 epochs: 500,
36 batch_size: 16,
37 shuffle: true,
38 seed: Some(3),
39 },
40 |epoch, loss| {
41 if epoch % 100 == 0 {
42 println!(" epoch {epoch:>3} loss {loss:.6}");
43 }
44 loss > 1e-5
45 },
46 )?;
47 let final_loss = model.evaluate_loss(&dataset)?;
48
49 println!("epochs run: {}", history.losses.len());
50 println!("final loss: {final_loss:.6}");
51 println!("[0.25, 0.50] -> {:.4}", model.predict(&[0.25, 0.50])?[0]);
52
53 Ok(())
54}Sourcepub fn from_csv(
path: impl AsRef<Path>,
target_columns: usize,
) -> Result<Self, NetworkError>
pub fn from_csv( path: impl AsRef<Path>, target_columns: usize, ) -> Result<Self, NetworkError>
Reads a comma-separated file in which the last target_columns columns
are the target and every column before them is an input.
A first row that does not parse as numbers is taken to be a header and
skipped. Quoted fields are understood, including "" for a literal
quote.
// sepal_length,sepal_width,petal_length,is_setosa
let dataset = Dataset::from_csv("iris.csv", 1)?;Sourcepub fn from_csv_labeled(
path: impl AsRef<Path>,
) -> Result<(Self, Vec<String>), NetworkError>
pub fn from_csv_labeled( path: impl AsRef<Path>, ) -> Result<(Self, Vec<String>), NetworkError>
Reads a comma-separated file whose last column is a class name rather than a number.
This is what a classification dataset off the internet actually looks
like — 5.1,3.5,1.4,0.2,setosa — and Dataset::from_csv rejects it,
because the last column does not parse as a float. The distinct labels
sorted become the classes, the target is a one-hot row over them, and
the names come back so a prediction can be named.
let (dataset, classes) = Dataset::from_csv_labeled(&path)?;
assert_eq!(classes, ["setosa", "versicolor"]);
assert_eq!(dataset.targets[0], vec![1.0, 0.0]);Sourcepub fn from_jsonl(
path: impl AsRef<Path>,
input_field: &str,
target_field: &str,
) -> Result<Self, NetworkError>
pub fn from_jsonl( path: impl AsRef<Path>, input_field: &str, target_field: &str, ) -> Result<Self, NetworkError>
Reads two NumPy .npy files, one of inputs and one of targets.
This is the file numpy.save writes, which is how data leaves PyTorch,
TensorFlow and scikit-learn without going through a CSV twenty times its
size. A 2-D array is one row per sample; a 1-D array is one value per
sample, which is what a column of labels looks like. float32,
float64, int32, int64 and uint8 all arrive as f32.
// numpy.save("x.npy", x); numpy.save("y.npy", y)
let mut dataset = Dataset::from_npy("x.npy", "y.npy")?;
dataset.one_hot_targets(10)?; // y held class indices, not one-hot rowsReads a whole JSONL file into memory, one row per line.
Each line is an object holding input_field and target_field, each
one an array of numbers or a single number. JsonlStream reads the
same format a batch at a time, for a file that does not fit in memory.
let dataset = Dataset::from_jsonl("corpus.jsonl", "pixels", "label")?;pub fn from_npy( inputs: impl AsRef<Path>, targets: impl AsRef<Path>, ) -> Result<Self, NetworkError>
Sourcepub fn from_idx(
images: impl AsRef<Path>,
labels: impl AsRef<Path>,
) -> Result<Self, NetworkError>
pub fn from_idx( images: impl AsRef<Path>, labels: impl AsRef<Path>, ) -> Result<Self, NetworkError>
Reads a pair of IDX files, the format MNIST and its many descendants ship in.
images is the idx3-ubyte file and labels the idx1-ubyte one. An
image of any shape is flattened into one input row, and u8 pixels are
scaled to 0.0..=1.0; labels are left as the class indices they are, so
one_hot_targets is the next call.
The files the archives ship are gzipped and this reads the plain ones:
gunzip train-images-idx3-ubyte.gz first.
let mut train = Dataset::from_idx("train-images-idx3-ubyte", "train-labels-idx1-ubyte")?;
train.one_hot_targets(10)?;Sourcepub fn one_hot_targets(&mut self, classes: usize) -> Result<(), NetworkError>
pub fn one_hot_targets(&mut self, classes: usize) -> Result<(), NetworkError>
Turns a single target column of class indices into one-hot rows over
classes classes.
Labels arrive as indices far more often than as one-hot rows — from
.npy, from a CSV of numbers — and a network with a softmax output
needs the row.
pub fn len(&self) -> usize
pub fn is_empty(&self) -> bool
pub fn shuffle(&mut self, seed: Option<u64>)
Sourcepub fn split_stratified(&self, train_ratio: f32) -> (Self, Self)
pub fn split_stratified(&self, train_ratio: f32) -> (Self, Self)
split, keeping each class’s share of the rows on
both sides.
A plain split of an imbalanced set can leave a rare class out of the test half entirely, and the accuracy it reports then says nothing about that class. Classes are read off the targets: the largest entry of a one-hot row, or the value itself when the target is a single column.
Row order inside a class is kept, so shuffle first — the split is otherwise the first rows of each class, which is whatever order the loader produced.
let mut dataset = Dataset::new(vec![vec![0.0]; 10], vec![vec![1.0, 0.0]; 10]);
dataset.targets[9] = vec![0.0, 1.0]; // one row of a rare class
let (train, test) = dataset.split_stratified(0.8);
assert_eq!((train.len(), test.len()), (8, 2));pub fn split(&self, train_ratio: f32) -> (Self, Self)
Sourcepub fn standardize(&mut self) -> Standardizer
pub fn standardize(&mut self) -> Standardizer
Centers every input column on zero and scales it to unit deviation, returning the statistics it used.
A network fed raw columns — an age beside an income — spends its first epochs undoing the scale difference, and with a saturating activation it may never recover. Fit on the training split and apply those statistics to the test split: fitting again on the test split leaks it.
let mut train = Dataset::new(vec![vec![1.0, 900.0], vec![3.0, 1100.0]], vec![vec![0.0]; 2]);
let mut test = Dataset::new(vec![vec![2.0, 1000.0]], vec![vec![1.0]]);
let statistics = train.standardize();
statistics.apply(&mut test);
assert_eq!(test.inputs[0], vec![0.0, 0.0]);Sourcepub fn flip_horizontal(&mut self, width: usize) -> Result<(), NetworkError>
pub fn flip_horizontal(&mut self, width: usize) -> Result<(), NetworkError>
Doubles the dataset with a left-right mirror of every input row.
A flipped photograph of a cat is still a photograph of a cat, so the
mirrored copy keeps its target and the model sees twice the data for
the price of one decode. Rows are the flat layout
from_image_folder produces — plane after
plane, each plane row-major of width pixels — so an RGB image flips
correctly with the same width as a grayscale one.
Apply it to the training split only. Augmenting before
split puts an image and its mirror on opposite
sides of the split and the test accuracy stops meaning anything.
let mut dataset = Dataset::new(vec![vec![1.0, 2.0, 3.0, 4.0]], vec![vec![1.0]]);
dataset.flip_horizontal(2)?; // two 2x1 rows
assert_eq!(dataset.inputs[1], vec![2.0, 1.0, 4.0, 3.0]);
assert_eq!(dataset.targets[1], vec![1.0]);This doubles the dataset once, up front. Random crops and rotations,
which want a fresh draw every epoch, are Augment.
Sourcepub fn stream(&mut self, config: &TrainConfig) -> DatasetStream<'_>
pub fn stream(&mut self, config: &TrainConfig) -> DatasetStream<'_>
Reads this dataset as a BatchSource, which is what
Network::fit trains from.
The stream takes the batch size, the shuffling and the seed from
config, and shuffles the dataset in place at the start of every epoch,
which is why it borrows mutably.
let mut dataset = Dataset::new(vec![vec![0.0]; 5], vec![vec![1.0]; 5]);
let config = TrainConfig { batch_size: 2, shuffle: false, ..Default::default() };
let mut stream = dataset.stream(&config);
stream.start_epoch(0)?;
let mut sizes = Vec::new();
while let Some(batch) = stream.next_batch()? {
sizes.push(batch.inputs.len());
}
assert_eq!(sizes, [2, 2, 1]);pub fn batches(&self, batch_size: usize) -> Vec<DatasetBatch<'_>>
Trait Implementations§
impl StructuralPartialEq for Dataset
Auto Trait Implementations§
impl Forget for Dataset
impl Freeze for Dataset
impl Move for Dataset
impl RefUnwindSafe for Dataset
impl Send for Dataset
impl Sync for Dataset
impl Unpin for Dataset
impl UnsafeUnpin for Dataset
impl UnwindSafe for Dataset
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more