use std::{error::Error, fmt};
use rusttorch_core::{Result, RustTorchError};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use crate::{TaskContext, Transform, TransformFactory, WorkerContext};
pub trait CheckpointableSource:
Iterator<Item = std::result::Result<crate::WorkerRecord<Self::Sample>, Self::Error>>
{
type Sample;
type State: Clone + Serialize + DeserializeOwned;
type Error;
fn error_sequence(&self, error: &Self::Error) -> crate::SequenceId;
fn snapshot(&self) -> Self::State;
fn validate_snapshot(&self, state: &Self::State) -> std::result::Result<(), Self::Error>;
fn restore_validated(&mut self, state: &Self::State);
fn set_run_context(&mut self, _context: WorkerContext) {}
}
pub trait CheckpointSourceFactory: crate::WorkerSourceFactory {
const CHECKPOINT_KIND: &'static str;
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct StreamCheckpointConfiguration {
pub workers: usize,
pub prefetch_factor: usize,
pub batch_size: usize,
pub drop_last: bool,
pub loader_seed: u64,
pub rank: usize,
pub pin_request: CheckpointPinRequest,
pub pin_status: CheckpointPinStatus,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct StreamLaneState<S, T> {
pub id: usize,
pub source: S,
pub transform: T,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct StreamLoaderState<S, T = (), C = ()> {
pub schema_version: u32,
pub source_identity: String,
pub factory_kind: String,
pub epoch: u64,
pub source_generation: u64,
pub transport_generation: u64,
pub next_batch: u64,
pub next_sequence: u64,
pub short_tail: bool,
pub source_length: Option<usize>,
pub rng_derivation_version: u32,
pub worker_seed_derivation_version: u32,
pub configuration: StreamCheckpointConfiguration,
pub lanes: Vec<StreamLaneState<S, T>>,
pub coordinator: C,
}
#[derive(Debug, thiserror::Error)]
pub enum StreamCheckpointBuildError<S, F> {
#[error(transparent)]
Configuration(#[from] RustTorchError),
#[error("stream source on worker {worker} failed: {source}")]
Source {
worker: usize,
#[source]
source: S,
},
#[error("stream transform initialization on worker {worker} failed: {source}")]
TransformFactory {
worker: usize,
#[source]
source: F,
},
#[error("stream transform validation on worker {worker} failed: {source}")]
TransformState {
worker: usize,
#[source]
source: RustTorchError,
},
#[error("stream checkpoint worker {worker} panicked")]
WorkerPanic {
worker: usize,
},
}
pub const LOADER_STATE_SCHEMA_VERSION: u32 = 1;
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum CheckpointPinRequest {
Disabled,
Auto,
ExplicitCuda {
index: usize,
},
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum CheckpointPinStatus {
Disabled,
DisabledNoAccelerator,
Cuda {
index: usize,
},
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct LoaderConfiguration {
pub batch_size: Option<usize>,
pub drop_last: bool,
pub workers: usize,
pub prefetch_factor: Option<usize>,
pub in_order: bool,
pub loader_seed: u64,
pub rank: usize,
pub replicas: usize,
pub distributed_shuffle: Option<bool>,
pub distributed_seed: Option<u64>,
pub distributed_drop_last: Option<bool>,
pub sampler_kind: String,
pub pin_request: CheckpointPinRequest,
pub pin_status: CheckpointPinStatus,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct LoaderState<D, S, T = (), C = ()> {
pub schema_version: u32,
pub dataset_identity: String,
pub epoch: u64,
pub iterator_generation: u64,
pub next_batch: u64,
pub next_logical_sample: u64,
pub dataset: D,
pub sampler: S,
pub transform: T,
pub collate: C,
pub rng_derivation_version: u32,
pub worker_seed_derivation_version: u32,
pub configuration: LoaderConfiguration,
}
pub trait Checkpointable {
type State: Clone + Serialize + DeserializeOwned;
fn save_state(&self) -> Self::State;
fn validate_state(&self, state: &Self::State) -> Result<()>;
fn load_validated(&mut self, state: &Self::State);
}
pub trait ReplaySafeDataset: crate::Dataset {}
pub trait DatasetCheckpoint: crate::Dataset {
type State: Clone + Serialize + DeserializeOwned;
fn snapshot_dataset(&self) -> Self::State;
fn validate_dataset_state(&self, state: &Self::State) -> Result<()>;
fn restore_dataset_validated(&mut self, state: &Self::State);
}
pub trait Stateless {}
pub trait TransactionalCheckpoint {
type State: Clone + Serialize + DeserializeOwned;
fn snapshot(&self) -> Self::State;
fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
fn restore_validated(&mut self, state: &Self::State);
}
pub trait WorkerCheckpoint {
type State: Clone + Serialize + DeserializeOwned;
fn snapshot(&self) -> Self::State;
fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
fn restore_validated(&mut self, state: &Self::State);
}
#[derive(Clone, Copy, Debug, Default)]
pub struct StatelessWorker<T> {
inner: T,
}
impl<T> StatelessWorker<T> {
pub fn new(inner: T) -> Self {
Self { inner }
}
pub fn into_inner(self) -> T {
self.inner
}
}
impl<Input, T> Transform<Input> for StatelessWorker<T>
where
T: Transform<Input>,
{
type Output = T::Output;
type Error = T::Error;
fn transform(
&mut self,
input: Input,
context: &TaskContext,
) -> std::result::Result<Self::Output, Self::Error> {
self.inner.transform(input, context)
}
}
impl<T> WorkerCheckpoint for StatelessWorker<T> {
type State = ();
fn snapshot(&self) -> Self::State {}
fn validate_snapshot(&self, _state: &Self::State) -> Result<()> {
Ok(())
}
fn restore_validated(&mut self, _state: &Self::State) {}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct TransactionalWorker<T> {
inner: T,
}
impl<T> TransactionalWorker<T> {
pub fn new(inner: T) -> Self {
Self { inner }
}
pub fn into_inner(self) -> T {
self.inner
}
}
impl<Input, T> Transform<Input> for TransactionalWorker<T>
where
T: Transform<Input> + TransactionalCheckpoint,
{
type Output = T::Output;
type Error = T::Error;
fn transform(
&mut self,
input: Input,
context: &TaskContext,
) -> std::result::Result<Self::Output, Self::Error> {
self.inner.transform(input, context)
}
}
impl<T> WorkerCheckpoint for TransactionalWorker<T>
where
T: TransactionalCheckpoint,
{
type State = T::State;
fn snapshot(&self) -> Self::State {
self.inner.snapshot()
}
fn validate_snapshot(&self, state: &Self::State) -> Result<()> {
self.inner.validate_snapshot(state)
}
fn restore_validated(&mut self, state: &Self::State) {
self.inner.restore_validated(state);
}
}
#[doc(hidden)]
#[derive(Clone, Copy, Debug)]
pub struct StatelessTransformFactory<F>(pub(crate) F);
impl<Input, F> TransformFactory<Input> for StatelessTransformFactory<F>
where
F: TransformFactory<Input>,
{
type Transform = StatelessWorker<F::Transform>;
type Error = F::Error;
fn create(
&self,
worker: Option<&WorkerContext>,
) -> std::result::Result<Self::Transform, Self::Error> {
self.0.create(worker).map(StatelessWorker::new)
}
}
#[doc(hidden)]
#[derive(Clone, Copy, Debug)]
pub struct TransactionalTransformFactory<F>(pub(crate) F);
impl<Input, F> TransformFactory<Input> for TransactionalTransformFactory<F>
where
F: TransformFactory<Input>,
F::Transform: TransactionalCheckpoint,
{
type Transform = TransactionalWorker<F::Transform>;
type Error = F::Error;
fn create(
&self,
worker: Option<&WorkerContext>,
) -> std::result::Result<Self::Transform, Self::Error> {
self.0.create(worker).map(TransactionalWorker::new)
}
}
#[derive(Debug)]
pub enum CheckpointBuildError<F> {
Configuration(RustTorchError),
TransformFactory(F),
}
impl<F> From<RustTorchError> for CheckpointBuildError<F> {
fn from(error: RustTorchError) -> Self {
Self::Configuration(error)
}
}
impl<F> fmt::Display for CheckpointBuildError<F>
where
F: fmt::Display,
{
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Configuration(error) => error.fmt(formatter),
Self::TransformFactory(error) => {
write!(
formatter,
"transform initialization failed while resuming: {error}"
)
}
}
}
}
impl<F> Error for CheckpointBuildError<F>
where
F: Error + 'static,
{
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Configuration(error) => Some(error),
Self::TransformFactory(error) => Some(error),
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct CheckpointDisabled;
#[derive(Clone, Debug)]
pub struct CheckpointFresh {
pub(crate) identity: String,
}
#[derive(Clone, Debug)]
pub struct CheckpointResume<S> {
pub(crate) identity: String,
pub(crate) state: S,
}
#[doc(hidden)]
pub struct PendingSerial<I, T> {
pub(crate) batches: I,
pub(crate) transform: T,
pub(crate) next_batch: u64,
pub(crate) next_logical_sample: u64,
}
#[doc(hidden)]
pub struct CheckpointActive<I, T> {
pub(crate) identity: String,
pub(crate) configuration: LoaderConfiguration,
pub(crate) pending: Option<PendingSerial<I, T>>,
}
#[doc(hidden)]
pub struct CheckpointIteration {
pub(crate) identity: String,
pub(crate) configuration: LoaderConfiguration,
pub(crate) boundary_valid: bool,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct WorkerLaneState<S> {
pub id: usize,
pub state: S,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub enum WorkerTransformLanes<S> {
Serial(S),
Workers(Vec<WorkerLaneState<S>>),
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct WorkerTransformState<S> {
pub schema_version: u32,
pub workers: usize,
pub prefetch_factor: Option<usize>,
pub factory_generation: u64,
pub run_generation: u64,
pub lanes: WorkerTransformLanes<S>,
}
#[doc(hidden)]
pub struct WorkerCheckpointActive<I, T, S> {
pub(crate) identity: String,
pub(crate) configuration: LoaderConfiguration,
pub(crate) batches: Option<I>,
pub(crate) serial: Option<T>,
pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
pub(crate) next_batch: u64,
pub(crate) next_logical_sample: u64,
pub(crate) factory_generation: u64,
pub(crate) run_generation: u64,
}
#[doc(hidden)]
pub struct WorkerCheckpointIteration<S, E> {
pub(crate) identity: String,
pub(crate) configuration: LoaderConfiguration,
pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
pub(crate) logical_ends: std::collections::BTreeMap<u64, u64>,
pub(crate) errors: std::collections::BTreeMap<u64, E>,
pub(crate) next_logical_sample: u64,
pub(crate) factory_generation: u64,
pub(crate) boundary_valid: bool,
}
impl Checkpointable for () {
type State = ();
fn save_state(&self) -> Self::State {}
fn validate_state(&self, _state: &Self::State) -> Result<()> {
Ok(())
}
fn load_validated(&mut self, _state: &Self::State) {}
}