pub struct CheckpointFresh { /* private fields */ }Expand description
Builder state after a dataset identity enables a fresh exact checkpoint.
Exact checkpointing requires explicit replay evidence; an arbitrary dataset is not enabled merely by assigning an identity:
use std::convert::Infallible;
use rusttorch_data::{DataLoader, Dataset, VecCollate};
struct Rows;
impl Dataset for Rows {
type Sample = i64;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
}
let _ = DataLoader::builder(Rows)
.collate(VecCollate)
.dataset_identity("rows".to_owned())
.build();Ordinary crate::TensorDataset rows share storage and are therefore not
exact-checkpoint capable until crate::TensorDataset::into_replay_safe
succeeds:
use rusttorch_core::Tensor;
use rusttorch_data::{DataLoader, TensorDataset, VecCollate};
let dataset = TensorDataset::new(vec![Tensor::from_slice(&[1_i64])]).unwrap();
let _ = DataLoader::builder(dataset)
.collate(VecCollate)
.dataset_identity("tensor".to_owned())
.build();An opaque closure transform has no restorable state contract unless the caller explicitly selects a stateless or transactional adapter:
use std::convert::Infallible;
use rusttorch_data::{
DataLoader, Dataset, FnTransform, ReplaySafeDataset, ReplaySafeMap,
TaskContext, VecCollate,
};
struct Rows;
impl Dataset for Rows {
type Sample = i64;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
}
impl ReplaySafeDataset for Rows {}
let transform = FnTransform::new(|value, _: &TaskContext| Ok::<_, Infallible>(value));
let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
.transform(transform)
.collate(VecCollate)
.dataset_identity("rows".to_owned())
.build();Closure-backed collation is likewise opaque without an explicit
Checkpointable implementation:
use std::convert::Infallible;
use rusttorch_data::{
DataLoader, Dataset, FnCollate, ReplaySafeDataset, ReplaySafeMap,
};
struct Rows;
impl Dataset for Rows {
type Sample = i64;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
}
impl ReplaySafeDataset for Rows {}
let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
.collate(FnCollate::new(|values| Ok::<_, Infallible>(values)))
.dataset_identity("rows".to_owned())
.build();Closure-backed sampler and explicit-batch sources do not expose stable configuration/cursor state:
use std::convert::Infallible;
use rusttorch_data::{
DataLoader, Dataset, FnSampler, ReplaySafeDataset, ReplaySafeMap,
VecCollate,
};
struct Rows;
impl Dataset for Rows {
type Sample = i64;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
}
impl ReplaySafeDataset for Rows {}
let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
.sampler(FnSampler::new(Some(1), |_| [0].into_iter()))
.collate(VecCollate)
.dataset_identity("rows".to_owned())
.build();use std::convert::Infallible;
use rusttorch_data::{
DataLoader, Dataset, FnBatchSource, ReplaySafeDataset, ReplaySafeMap,
VecCollate,
};
struct Rows;
impl Dataset for Rows {
type Sample = i64;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
}
impl ReplaySafeDataset for Rows {}
let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
.batch_sampler(FnBatchSource::new(Some(1), |_| vec![vec![0]].into_iter()))
.collate(VecCollate)
.dataset_identity("rows".to_owned())
.build();Worker replay supports explicit immutable datasets and checkpointable transforms. Zero workers use the same tagged envelope:
use std::convert::Infallible;
use rusttorch_data::{
DataLoader, Dataset, ReplaySafeDataset, ReplaySafeMap, VecCollate,
};
struct Rows;
impl Dataset for Rows {
type Sample = i64;
type Error = Infallible;
fn len(&self) -> usize { 1 }
fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
}
impl ReplaySafeDataset for Rows {}
let mut loader = DataLoader::builder(ReplaySafeMap::new(Rows))
.workers(0)
.collate(VecCollate)
.dataset_identity("rows".to_owned())
.build().unwrap();
let state = loader.iter().checkpoint().unwrap();
let mut resumed = DataLoader::builder(ReplaySafeMap::new(Rows))
.workers(0).collate(VecCollate).dataset_identity("rows".to_owned())
.resume_from(state).build().unwrap();
assert_eq!(resumed.iter().next().unwrap().unwrap(), vec![1]);Trait Implementations§
Source§impl Clone for CheckpointFresh
impl Clone for CheckpointFresh
Source§fn clone(&self) -> CheckpointFresh
fn clone(&self) -> CheckpointFresh
1.0.0 (const: unstable) · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
source. Read more