nncombinator 0.9.8

//! Implementation of all full connected layers
use std::fmt::Debug;
use std::marker::PhantomData;
use std::str::FromStr;
use crate::arr::{Arr, Arr2, IntoConverter};
use crate::{Cons, Stack};
use crate::device::{Device, DeviceBatchAveraging};
use crate::device::linear::{DeviceDiffLinear, DeviceLinear};
use crate::error::{ModelLoadError, EvaluateError, LayerInstantiationError, PersistenceError, TrainingError, TypeConvertError};
use crate::layer::{Backward, BackwardAll, BatchBackward, BatchDataType, BatchForward, BatchForwardBase, BatchLoss, BatchPreTrain, BatchPreTrainBase, BatchSize, ContinueForward, Forward, ForwardAll, ForwardDiff, Loss, PartialForward, PreTrain, UpdateWeight, OnStep, PersistProgress};
use crate::lossfunction::LossFunction;
use crate::ope::UnitValue;
use crate::optimizer::{Optimizer, OptimizerBuilder};
use crate::persistence::{Linear, LinearPersistence, Persistence, Specialized, TextFilePersistence, TextPersistence, TextRecord};

/// Linear Layer Implementation
pub struct LinearLayer<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          D: Device<U>,
          I: Debug + Send + Sync,
          PI: Debug,
          OP: Optimizer<U,D> {
    u:PhantomData<U>,
    parent:P,
    device:D,
    units:C,
    bias:BC,
    unit_optimizer:OP,
    bias_optimizer:OP,
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> + PreTrain<U> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    /// Create and return an instance of LinearLayer
    /// # Arguments
    /// * `parent` - upper layer
    /// * `device` - Device object used for neural network computation
    /// * `ui` - Callback to generate weight of unit
    /// * `bi` - Callback to generate weight of bias
    /// * `b` - optimizer builder
    pub fn new<UI,BI,B>(parent:P,device:&D,mut ui:UI,mut bi:BI, b:&B)
        -> Result<LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>,LayerInstantiationError>
        where UI: FnMut() -> U, BI: FnMut() -> U, B: OptimizerBuilder<U,D,Output=OP> {

        let mut units:Arr2<U,NI,NO> = Arr2::new();
        let mut bias:Arr<U,NO> = Arr::new();

        for mut it in units.iter_mut() {
            for it in it.iter_mut() {
                *it = ui();
            }
        }

        for it in bias.iter_mut() {
            *it = bi();
        }

        let units = device.specialization_units(units)?;
        let bias = device.specialization_bias(bias)?;

        Ok(LinearLayer {
            u:PhantomData::<U>,
            parent:parent,
            device:device.clone(),
            units: units,
            bias:bias,
            unit_optimizer:b.build(NI*NO)?,
            bias_optimizer:b.build(NO)?
        })
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> Persistence<U,TextFilePersistence,Specialized> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U> + Loss<U> + Persistence<U,TextFilePersistence,Specialized>,
          U: Default + Clone + Copy + UnitValue<U> + FromStr,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          TextRecord: From<U>,
          ModelLoadError: From<<U as FromStr>::Err> {
    fn load(&mut self, persistence: &mut TextFilePersistence) -> Result<(), ModelLoadError> {
        self.parent.load(persistence)?;

        let mut bias = Arr::<U,NO>::new();

        for b in bias.iter_mut() {
            *b = persistence.read()?;
        }

        let mut units = Arr2::<U,NI,NO>::new();

        for mut u in units.iter_mut() {
            for w in u.iter_mut() {
                *w = persistence.read()?;
            }
        }

        self.units = self.device.specialization_units(units)?;
        self.bias = self.device.specialization_bias(bias)?;

        Ok(())
    }

    fn save(&mut self, persistence: &mut TextFilePersistence) -> Result<(), PersistenceError> {
        self.parent.save(persistence)?;

        persistence.write_layer_start();

        let bias = self.device.generalization_bias(&self.bias)?;
        let units = self.device.generalization_units(&self.units)?;

        persistence.write_units_start();

        for b in bias.iter() {
            persistence.write(*b);
        }

        for u in units.iter() {
            persistence.write_units_start();
            for w in u.iter() {
                persistence.write(*w);
            }
        }

        persistence.write_layer_end();

        Ok(())
    }
}
impl<T,U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> Persistence<U,T,Linear> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where T: LinearPersistence<U>,
          P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U> + Loss<U> + Persistence<U,T,Linear>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    fn load(&mut self, persistence: &mut T) -> Result<(), ModelLoadError> {
        self.parent.load(persistence)?;

        let mut bias = Arr::<U,NO>::new();

        for b in bias.iter_mut() {
            *b = persistence.read()?;
        }

        let mut units = Arr2::<U,NI,NO>::new();

        for mut u in units.iter_mut() {
            for w in u.iter_mut() {
                *w = persistence.read()?;
            }
        }

        self.units = self.device.specialization_units(units)?;
        self.bias = self.device.specialization_bias(bias)?;

        Ok(())
    }

    fn save(&mut self, persistence: &mut T) -> Result<(), PersistenceError> {
        self.parent.save(persistence)?;

        let bias = self.device.generalization_bias(&self.bias)?;
        let units = self.device.generalization_units(&self.units)?;

        for b in bias.iter() {
            persistence.write(*b)?;
        }

        for u in units.iter() {
            for w in u.iter() {
                persistence.write(*w)?;
            }
        }

        Ok(())
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> Forward<PI,Result<<Self as ForwardAll>::Output,EvaluateError>>
    for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {

    fn forward(&self,input:&PI) -> Result<<Self as ForwardAll>::Output,EvaluateError> {
        self.device.forward_linear(&self.bias,&self.units,input.into())
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> ForwardAll for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    type Input = I;
    type Output = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output;
    fn forward_all(&self, input: Self::Input) -> Result<Self::Output, EvaluateError> {
        self.forward(&self.parent.forward_all(input)?)
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> PreTrain<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: PreTrain<U,PreOutput=PI> +
             ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          OP: Optimizer<U,D>,
          PI: Debug + BatchDataType + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    type PreOutput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output;
    type OutStack = Cons<<P as PreTrain<U>>::OutStack,Self::PreOutput>;

    fn pre_train(&self, input: Self::Input) -> Result<Self::OutStack, EvaluateError> {
        let r = self.parent.pre_train(input)?;

        let u = r.map(|r| self.forward(r))?;

        Ok(Cons(r,u))
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize>
    Backward<U,&<D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output,Result<PI,TrainingError>> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          OP: Optimizer<U,D>,
          PI: Debug + BatchDataType + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    fn backward(&mut self, input: &<D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output) -> Result<PI,TrainingError> {
        Ok(self.device.backward_linear(&self.units,input)?.into())
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> BackwardAll<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: BackwardAll<U,LossInput=PI> + ForwardAll<Input=I,Output=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          C: Debug,
          BC: Debug,
          OP: Optimizer<U,D>,
          PI: Debug + BatchDataType +
              From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> + DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a C>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a BC>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut C>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut BC> {
    type LossInput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output;
    type LossOutput = <P as BackwardAll<U>>::LossOutput;

    fn backward_all<L: LossFunction<U>>(&mut self, input: Self::LossInput, stack:Self::OutStack, lossf:&L)
        -> Result<(<Self as BackwardAll<U>>::LossOutput,<Self as UpdateWeight<U>>::GradientStack), TrainingError> {
        let (s,_) = stack.pop();

        let loss = input;

        let next_loss = self.backward(&loss)?;

        let g = s.map(|o| {
            self.device.backward_weight_gradient(o.into(),&loss)
        })?;

        let bg = self.device.backward_bias_weight_gradient(loss)?;

        let (s,loss) = self.parent.loss(next_loss.into(),lossf,s)?;

        let (l,s) = self.parent.backward_all(loss, s, lossf)?;

        Ok((l,Cons(s,(g,bg))))
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> PartialForward for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + PartialForward<DiffOutput=PI> +
             BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          Self: ForwardAll<Input=I>,
          Self: PreTrain<U> {
    type PartialOutput = <P as PartialForward>::PartialOutput;
    type PartialOutputByDiff = <P as PartialForward>::PartialOutputByDiff;
    type DiffInput = <P as PartialForward>::DiffInput;
    type DiffOutput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output;

    fn partial_forward(&self, input: Self::Input) -> Result<Self::PartialOutput, EvaluateError> {
        Ok(self.parent.partial_forward(input)?)
    }

    fn partial_forward_by_diff(&self, input: Self::DiffInput) -> Result<Self::PartialOutputByDiff, EvaluateError> {
        Ok(self.parent.partial_forward_by_diff(input)?)
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> ForwardDiff for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + PartialForward<DiffOutput=PI> + ForwardDiff +
             BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          Self: ForwardAll<Input=I> + PreTrain<U> {
    fn forward_diff(&self, input: Self::DiffInput) -> Result<Self::DiffOutput, EvaluateError> {
        let input = self.parent.forward_diff(input)?;

        Ok(self.device.forward_linear(&self.bias,&self.units,&input.into())?)
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> ContinueForward for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + PartialForward<DiffOutput=PI> + ContinueForward<ConinueOutput=PI> +
             BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          Self: ForwardAll<Input=I,Output=<D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output> + PreTrain<U> {
    type ConinueOutput = Self::Output;
    fn continue_forward(&self, input: &Self::PartialOutput) -> Result<Self::ConinueOutput, EvaluateError> {
        let input = self.parent.continue_forward(input)?;

        Ok(self.device.forward_linear(&self.bias,&self.units,&input.into())?)
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> UpdateWeight<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> +
             Loss<U> + UpdateWeight<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          C: Debug,
          BC: Debug,
          OP: Optimizer<U,D>,
          PI: Debug + BatchDataType + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> + DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a C>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a BC>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut C>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut BC> {
    type GradientStack = Cons<<P as UpdateWeight<U>>::GradientStack,(C,BC)>;

    fn update_weight(&mut self, stack: Self::GradientStack, batch_size: usize) -> Result<(), TrainingError> {
        let (s,(g,bg)) = stack.pop();

        let g = self.device.batch_averaging(g,batch_size)?;
        let bg = self.device.batch_averaging(bg,batch_size)?;

        self.bias_optimizer.update((&bg).into(), (&mut self.bias).into())?;
        self.unit_optimizer.update((&g).into(),(&mut self.units).into())?;

        Ok(self.parent.update_weight(s,batch_size)?)
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> Loss<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: PreTrain<U,PreOutput=PI> +
             ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> + Loss<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          C: Debug,
          BC: Debug,
          OP: Optimizer<U,D>,
          PI: Debug + BatchDataType + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> + DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a C>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a BC>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut C>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut BC> {
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> BatchForwardBase for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> +
             Loss<U> + BatchForwardBase<BatchInput=<I as BatchDataType>::Type,BatchOutput=<PI as BatchDataType>::Type>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync + BatchDataType,
          <I as BatchDataType>::Type: Debug,
          PI: Debug + BatchDataType + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput: Debug {
    type BatchInput = <I as BatchDataType>::Type;
    type BatchOutput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput;
}
impl<U,C,BC,P,OP,D,I,PI,const NI:usize,const NO:usize> BatchForward for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U> +
             BatchForwardBase<BatchInput=<I as BatchDataType>::Type,BatchOutput=<PI as BatchDataType>::Type> + BatchForward,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync + BatchDataType,
          OP: Optimizer<U,D>,
          <I as BatchDataType>::Type: Debug,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          PI: Debug + BatchDataType + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput> + BatchDataType,
          <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput: Debug {
    fn batch_forward(&self, input: Self::BatchInput) -> Result<Self::BatchOutput, TrainingError> {
        let input = self.parent.batch_forward(input)?;

        Ok(self.device.batch_forward_linear(&self.bias,&self.units,&input)?)
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> BatchPreTrainBase<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> + PreTrain<U,PreOutput=PI> + Loss<U> +
             BatchForwardBase<BatchInput=<I as BatchDataType>::Type,BatchOutput=<PI as BatchDataType>::Type> + BatchForward +
             BatchPreTrainBase<U,BatchPreOutput=<PI as BatchDataType>::Type>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync + BatchDataType,
          OP: Optimizer<U,D>,
          <PI as BatchDataType>::Type: BatchSize,
          <I as BatchDataType>::Type: Debug,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          PI: Debug + BatchDataType + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput> + BatchDataType,
          <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput: Debug,
          Self: PreTrain<U,PreOutput=<D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output> {
    type BatchPreOutput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput;
    type BatchOutStack = Cons<<P as BatchPreTrainBase<U>>::BatchOutStack,Self::BatchPreOutput>;
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> BatchPreTrain<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U> +
             BatchForwardBase<BatchInput=<I as BatchDataType>::Type,BatchOutput=<PI as BatchDataType>::Type> +
             BatchForward +
             BatchPreTrainBase<U,BatchPreOutput=<PI as BatchDataType>::Type> +
             BatchPreTrain<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync + BatchDataType,
          OP: Optimizer<U,D>,
          <I as BatchDataType>::Type: Debug,
          <PI as BatchDataType>::Type: BatchSize,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          PI: Debug + BatchDataType + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput> + BatchDataType,
          <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput: Debug {
    fn batch_pre_train(&self, input: Self::BatchInput) -> Result<Self::BatchOutStack, TrainingError> {
        let r = self.parent.batch_pre_train(input)?;

        let u = r.map(|input| {
            self.device.batch_forward_linear(&self.bias,&self.units,input)
        })?;

        Ok(Cons(r,u))
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> BatchBackward<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> +
             Loss<U> + BatchForwardBase<BatchInput=<I as BatchDataType>::Type,BatchOutput=<PI as BatchDataType>::Type> + BatchForward +
             BatchPreTrainBase<U,BatchPreOutput=<PI as BatchDataType>::Type> +
             BatchPreTrain<U> +
             BatchBackward<U> + BatchLoss<U,BatchLossInput=<PI as BatchDataType>::Type>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync + BatchDataType,
          C: Debug,
          BC: Debug,
          PI: BatchDataType,
          OP: Optimizer<U,D>,
          <I as BatchDataType>::Type: Debug,
          <PI as BatchDataType>::Type: BatchSize,
          <PI as BatchDataType>::Type: IntoConverter,
          <PI as BatchDataType>::Type: TryFrom<<<PI as BatchDataType>::Type as IntoConverter>::Converter,Error=TypeConvertError> + Debug,
          PI: Debug + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput> + BatchDataType,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO,BatchLossOutput=<PI as BatchDataType>::Type> +
             DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput: Debug,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a C>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a BC>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut C>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut BC> {
    type BatchLossInput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput;
    type BatchLossOutput = <P as BatchBackward<U>>::BatchLossOutput;

    fn batch_backward<L: LossFunction<U>>(&mut self, input: Self::BatchLossInput, stack: Self::BatchOutStack, lossf: &L)
        -> Result<(<Self as BatchBackward<U>>::BatchLossOutput,<Self as UpdateWeight<U>>::GradientStack), TrainingError> {
        let (s, _) = stack.pop();

        let loss = input;

        let next_loss = self.device.batch_backward_linear(&self.units, &loss)?;

        let g = s.map(|o| {
            self.device.batch_backward_weight_gradient(o, &loss)
        })?;

        let bg = self.device.batch_linear_reduce(&loss)?;

        let (
            s,loss
        ) = self.parent.batch_loss(next_loss.into_converter().try_into()?,lossf,s)?;

        let (l,s) = self.parent.batch_backward(loss, s, lossf)?;

        Ok((l,Cons(s,(g,bg))))
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> BatchLoss<U> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U> +
             BatchForwardBase<BatchInput=<I as BatchDataType>::Type,BatchOutput=<PI as BatchDataType>::Type> + BatchForward +
             BatchPreTrainBase<U,BatchPreOutput=<PI as BatchDataType>::Type> +
             BatchPreTrain<U> +
             BatchBackward<U> + BatchLoss<U,BatchLossInput=<PI as BatchDataType>::Type>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync + BatchDataType,
          C: Debug,
          BC: Debug,
          OP: Optimizer<U,D>,
          PI: Debug + From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::LossOutput> + BatchDataType,
          <I as BatchDataType>::Type: Debug,
          <PI as BatchDataType>::Type: BatchSize,
          <PI as BatchDataType>::Type: IntoConverter,
          <PI as BatchDataType>::Type: TryFrom<<<PI as BatchDataType>::Type as IntoConverter>::Converter,Error=TypeConvertError> + Debug,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO,BatchLossOutput=<PI as BatchDataType>::Type> +
             DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          <D as DeviceLinear<U,C,BC,PI,NI,NO>>::BatchOutput: Debug,
          <<D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output as BatchDataType>::Type: Debug,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a C>,
          for<'a> &'a <OP as Optimizer<U,D>>::InternalType: From<&'a BC>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut C>,
          for<'a> <OP as Optimizer<U,D>>::InternalUpdateType<'a>: From<&'a mut BC> {
}
// OnStep implementation for LinearLayer
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> OnStep for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> + PreTrain<U> + Loss<U> + OnStep,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          D: Device<U>,
          I: Debug + Send + Sync,
          PI: Debug,
          OP: Optimizer<U,D> {
    fn on_step(&mut self, step: usize) -> Result<(), TrainingError> {
        self.unit_optimizer.on_step(step)?;
        self.bias_optimizer.on_step(step)?;
        Ok(self.parent.on_step(step)?)
    }
    fn on_frequently_step(&mut self, step: usize, frequently_step: usize) -> Result<(), TrainingError> {
        self.unit_optimizer.on_frequently_step(step,frequently_step)?;
        self.bias_optimizer.on_frequently_step(step,frequently_step)?;
        Ok(self.parent.on_frequently_step(step,frequently_step)?)
    }
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> PersistProgress<TextFilePersistence,Specialized> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U> + Loss<U> +
             PersistProgress<TextFilePersistence,Specialized>,
          U: Default + Clone + Copy + UnitValue<U> + FromStr,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D> + Persistence<U,TextFilePersistence,Specialized>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          TextRecord: From<U>,
          ModelLoadError: From<<U as FromStr>::Err> {
    fn load_progress(&mut self, persistence: &mut TextFilePersistence) -> Result<(), TrainingError> {
        self.parent.load_progress(persistence)?;

        self.unit_optimizer.load(persistence)?;
        self.bias_optimizer.load(persistence)?;

        Ok(())
    }

    fn save_progress(&mut self, persistence: &mut TextFilePersistence) -> Result<(), PersistenceError> {
        self.parent.save_progress(persistence)?;

        persistence.write_layer_start();

        persistence.write_units_start();

        self.unit_optimizer.save(persistence)?;
        persistence.write_units_start();

        self.bias_optimizer.save(persistence)?;

        persistence.write_layer_end();

        Ok(())
    }
}
impl<T,U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> PersistProgress<T,Linear> for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where T: LinearPersistence<U>,
          P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U> + Loss<U> +
             PersistProgress<T,Linear>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D> + Persistence<U,T,Linear>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    fn load_progress(&mut self, persistence: &mut T) -> Result<(), TrainingError> {
        self.parent.load_progress(persistence)?;

        self.unit_optimizer.load(persistence)?;
        self.bias_optimizer.load(persistence)?;

        Ok(())
    }

    fn save_progress(&mut self, persistence: &mut T) -> Result<(), PersistenceError> {
        self.parent.save_progress(persistence)?;

        self.unit_optimizer.save(persistence)?;
        self.bias_optimizer.save(persistence)?;

        Ok(())
    }
}
/// Trait for LinearLayer instance creation
pub trait LinearLayerInstantiation<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug,
          OP: Optimizer<U,D>,
          D: Device<U> {
    /// Create an instance of LinearLayers
    /// # Arguments
    /// * `parent` - upper layer
    /// * `device` - Device object used for neural network computation
    /// * `ui` - Callback to generate weight of unit
    /// * `bi` - Callback to generate weight of bias
    /// * `b` - optimizer builder
    ///
    /// # Errors
    ///
    /// This function may return the following errors
    /// * [`LayerInstantiationError`]
    fn instantiation<B: OptimizerBuilder<U,D,Output=OP>>(parent:P,device:&D,ui: impl FnMut() -> U, bi: impl FnMut() -> U, b: &B)
        -> Result<LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>,LayerInstantiationError>;
}
impl<U,C,BC,P,D,I,PI,OP,const NI:usize,const NO:usize> LinearLayerInstantiation<U,C,BC,P,D,I,PI,OP,NI,NO>
    for LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    fn instantiation<B: OptimizerBuilder<U,D,Output=OP>>(parent: P, device:&D, ui: impl FnMut() -> U, bi: impl FnMut() -> U, b: &B)
        -> Result<LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>,LayerInstantiationError> {
        Ok(LinearLayer::<_,_,_,_,_,_,_,_,NI,NO>::new(parent,device,ui,bi,b)?)
    }
}
/// Builder for LinearLayer instance creation
pub struct LinearLayerBuilder<const NI:usize,const NO:usize> {
    ni:PhantomData<[();NI]>,
    no:PhantomData<[();NO]>
}
impl<const NI:usize,const NO:usize> LinearLayerBuilder<NI,NO> {
    /// Create an instance of LinearLayerBuilder
    pub fn new() -> LinearLayerBuilder<NI,NO> {
        LinearLayerBuilder {
            ni:PhantomData::<[();NI]>,
            no:PhantomData::<[();NO]>
        }
    }
}
impl<const NI:usize,const NO:usize> LinearLayerBuilder<NI,NO> {
    /// Create an instance of LinearLayers
    /// # Arguments
    /// * `parent` - upper layer
    /// * `device` - Device object used for neural network computation
    /// * `ui` - Callback to generate weight of unit
    /// * `bi` - Callback to generate weight of bias
    ///
    /// # Errors
    ///
    /// This function may return the following errors
    /// * [`LayerInstantiationError`]
    pub fn build<U,C,BC,P,D,I,PI,OP,B>(&self,parent: P, device:&D, ui: impl FnMut() -> U, bi: impl FnMut() -> U, b: &B)
        -> Result<LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>,LayerInstantiationError>
        where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=PI> +
                 PreTrain<U,PreOutput=PI> + Loss<U>,
              U: Default + Clone + Copy + UnitValue<U>,
              I: Debug + Send + Sync,
              PI: Debug,
              OP: Optimizer<U,D>,
              B: OptimizerBuilder<U,D,Output=OP>,
              D: Device<U>,
              LinearLayer<U,C,BC,P,D,I,PI,OP,NI,NO>: LinearLayerInstantiation<U,C,BC,P,D,I,PI,OP,NI,NO> {

        LinearLayer::instantiation(parent,device,ui,bi,b)
    }
}
/// Implementation of differentially applicable linear layers
pub struct DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize>
    where P: ForwardAll<Input=I,Output=PI> + PreTrain<U> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          D: Device<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug,
          OP: Optimizer<U,D> {
    u:PhantomData<U>,
    pi:PhantomData<PI>,
    di:PhantomData<DI>,
    l:PhantomData<&'a ()>,
    parent:P,
    device:D,
    units:C,
    bias:BC,
    unit_optimizer: OP,
    bias_optimizer: OP
}
impl<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> + PreTrain<U> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          DI: Debug,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    /// Create and return an instance of DiffLinearLayer
    /// # Arguments
    /// * `parent` - upper layer
    /// * `device` - Device object used for neural network computation
    /// * `ui` - Callback to generate weight of unit
    /// * `bi` - Callback to generate weight of bias
    /// * `b` - optimizer builder
    pub fn new<UI,BI,B>(parent:P,device:&D,mut ui:UI,mut bi:BI, b: &B)
        -> Result<DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>,LayerInstantiationError>
        where UI: FnMut() -> U, BI: FnMut() -> U, B: OptimizerBuilder<U,D,Output=OP> {

        let mut units:Arr2<U,NI,NO> = Arr2::new();
        let mut bias:Arr<U,NO> = Arr::new();

        for mut it in units.iter_mut() {
            for it in it.iter_mut() {
                *it = ui();
            }
        }

        for it in bias.iter_mut() {
            *it = bi();
        }

        let units = device.specialization_units(units)?;
        let bias = device.specialization_bias(bias)?;

        Ok(DiffLinearLayer {
            u:PhantomData::<U>,
            pi:PhantomData::<PI>,
            di:PhantomData::<DI>,
            l:PhantomData::<&'a ()>,
            parent:parent,
            device:device.clone(),
            units: units,
            bias:bias,
            unit_optimizer:b.build(NI*NO)?,
            bias_optimizer:b.build(NO)?
        })
    }
}
impl<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> Persistence<U,TextFilePersistence,Specialized>
    for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> + PreTrain<U> + Loss<U> +
             Persistence<U,TextFilePersistence,Specialized>,
          U: Default + Clone + Copy + UnitValue<U> + FromStr,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          DI: Debug,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          TextRecord: From<U>,
          ModelLoadError: From<<U as FromStr>::Err> {
    fn load(&mut self, persistence: &mut TextFilePersistence) -> Result<(), ModelLoadError> {
        self.parent.load(persistence)?;

        let mut bias = Arr::<U,NO>::new();

        for b in bias.iter_mut() {
            *b = persistence.read()?;
        }

        let mut units = Arr2::<U,NI,NO>::new();

        for mut u in units.iter_mut() {
            for w in u.iter_mut() {
                *w = persistence.read()?;
            }
        }

        self.bias = self.device.specialization_bias(bias)?;
        self.units = self.device.specialization_units(units)?;

        Ok(())
    }

    fn save(&mut self, persistence: &mut TextFilePersistence) -> Result<(), PersistenceError> {
        self.parent.save(persistence)?;

        persistence.write_layer_start();

        let bias = self.device.generalization_bias(&self.bias)?;
        let units = self.device.generalization_units(&self.units)?;

        persistence.write_units_start();

        for b in bias.iter() {
            persistence.write(*b);
        }

        for u in units.iter() {
            persistence.write_units_start();
            for w in u.iter() {
                persistence.write(*w);
            }
        }

        persistence.write_layer_end();

        Ok(())
    }
}
impl<'a,T,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> Persistence<U,T,Linear>
    for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where T: LinearPersistence<U>,
          P: ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> + PreTrain<U> + Loss<U> +
             Persistence<U,T,Linear>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          DI: Debug,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    fn load(&mut self, persistence: &mut T) -> Result<(), ModelLoadError> {
        self.parent.load(persistence)?;

        let mut bias = Arr::<U,NO>::new();

        for b in bias.iter_mut() {
            *b = persistence.read()?;
        }

        let mut units = Arr2::<U,NI,NO>::new();

        for mut u in units.iter_mut() {
            for w in u.iter_mut() {
                *w = persistence.read()?;
            }
        }

        self.bias = self.device.specialization_bias(bias)?;
        self.units = self.device.specialization_units(units)?;

        Ok(())
    }

    fn save(&mut self, persistence: &mut T) -> Result<(), PersistenceError> {
        self.parent.save(persistence)?;

        let bias = self.device.generalization_bias(&self.bias)?;
        let units = self.device.generalization_units(&self.units)?;

        for b in bias.iter() {
            persistence.write(*b)?;
        }

        for u in units.iter() {
            for w in u.iter() {
                persistence.write(*w)?;
            }
        }

        Ok(())
    }
}
impl<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> ForwardAll for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceDiffLinear<'a,U,DI,C,NI,NO> + DeviceLinear<U,C,BC,PI,NI,NO> {
    type Input = I;
    type Output = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output;

    fn forward_all(&self, input: Self::Input) -> Result<Self::Output, EvaluateError> {
        let input = self.parent.forward_all(input)?;

        Ok(self.device.forward_linear(&self.bias,&self.units,&input)?)
    }
}
impl<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> PreTrain<U> for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: PreTrain<U,PreOutput=PI> +
             ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceDiffLinear<'a,U,DI,C,NI,NO> + DeviceLinear<U,C,BC,PI,NI,NO>,
          <D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output: Debug + 'static {
    type PreOutput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output;
    type OutStack = Cons<<P as PreTrain<U>>::OutStack,Self::PreOutput>;

    fn pre_train(&self, input: Self::Input) -> Result<Self::OutStack, EvaluateError> {
        let s = self.parent.pre_train(input)?;

        let u = s.map(|input| {
            self.device.forward_linear(&self.bias,&self.units,input)
        })?;

        Ok(Cons(s,u))
    }
}
impl<'a,U,C,BC,P,D,OP,I,DI,PI,const NI:usize,const NO:usize> BackwardAll<U> for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: BackwardAll<U,LossInput=()> +
             ForwardAll<Input=I,Output=PI> +
             PreTrain<U,PreOutput=PI> +
             Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          C: Debug,
          BC: Debug,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceDiffLinear<'a,U,DI,C,NI,NO> + DeviceLinear<U,C,BC,PI,NI,NO> +
             DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output: Debug + 'static,
          BC: From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output>,
          for<'b> &'b <OP as Optimizer<U,D>>::InternalType: From<&'b C>,
          for<'b> &'b <OP as Optimizer<U,D>>::InternalType: From<&'b BC>,
          for<'b> <OP as Optimizer<U,D>>::InternalUpdateType<'b>: From<&'b mut C>,
          for<'b> <OP as Optimizer<U,D>>::InternalUpdateType<'b>: From<&'b mut BC>,
          Self: ForwardAll + PreTrain<U,OutStack=Cons<<P as PreTrain<U>>::OutStack,BC>> {
    type LossInput = <D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output;
    type LossOutput = <P as BackwardAll<U>>::LossOutput;

    fn backward_all<L: LossFunction<U>>(&mut self, input: Self::LossInput, stack:Self::OutStack, lossf:&L)
        -> Result<(<Self as BackwardAll<U>>::LossOutput,<Self as UpdateWeight<U>>::GradientStack), TrainingError> {
        let (s,_) = stack.pop();

        let loss = input;

        let g = s.map(|o| {
            self.device.backward_weight_gradient(o,&loss)
        })?;

        let bg = loss;

        let (l,s) = self.parent.backward_all((), s, lossf)?;

        Ok((l,Cons(s,(g,bg.into()))))
    }
}
impl<'a,U,C,BC,P,D,OP,I,DI,PI,const NI:usize,const NO:usize> UpdateWeight<U> for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> +
             PreTrain<U> +
             Loss<U> + UpdateWeight<U>,
          U: Default + Clone + Copy + Send + UnitValue<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          C: Debug,
          BC: Debug,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          for<'b> &'b <OP as Optimizer<U,D>>::InternalType: From<&'b C>,
          for<'b> &'b <OP as Optimizer<U,D>>::InternalType: From<&'b BC>,
          for<'b> <OP as Optimizer<U,D>>::InternalUpdateType<'b>: From<&'b mut C>,
          for<'b> <OP as Optimizer<U,D>>::InternalUpdateType<'b>: From<&'b mut BC> {
    type GradientStack = Cons<<P as UpdateWeight<U>>::GradientStack,(C,BC)>;

    fn update_weight(&mut self, stack: Self::GradientStack, batch_size: usize) -> Result<(), TrainingError> {
        let (s,(g,bg)) = stack.pop();

        let g = self.device.batch_averaging(g,batch_size)?;
        let bg = self.device.batch_averaging(bg,batch_size)?;

        self.bias_optimizer.update((&bg).into(),(&mut self.bias).into())?;
        self.unit_optimizer.update((&g).into(),(&mut self.units).into())?;

        Ok(self.parent.update_weight(s,batch_size)?)
    }
}
impl<'a,U,C,BC,P,D,OP,I,DI,PI,const NI:usize,const NO:usize> PartialForward for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: BackwardAll<U,LossInput=()> +
             ForwardAll<Input=I,Output=PI> +
             PartialForward<PartialOutput=PI,DiffInput=DI,DiffOutput=DI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO,Output=<D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output> +
             DeviceDiffLinear<'a,U,DI,C,NI,NO>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          Self: ForwardAll<Input=I> +
                PreTrain<U> {
    type PartialOutput = <D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output;
    type PartialOutputByDiff = <D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output;
    type DiffInput = DI;
    type DiffOutput = <D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output;

    fn partial_forward(&self, input: Self::Input) -> Result<Self::PartialOutput, EvaluateError> {
        let input = self.parent.partial_forward(input)?;

        Ok(self.device.forward_linear(&self.bias,&self.units,&input)?)
    }

    fn partial_forward_by_diff(&self, input: Self::DiffInput) -> Result<Self::PartialOutputByDiff, EvaluateError> {
        Ok(self.device.forward_diff_linear(&self.units,input)?)
    }
}
impl<'a,U,C,BC,P,D,OP,I,DI,PI,const NI:usize,const NO:usize> ForwardDiff for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: BackwardAll<U,LossInput=()> +
             ForwardAll<Input=I,Output=PI> + PartialForward<PartialOutput=PI> +
             ForwardDiff +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          D: Device<U> + DeviceDiffLinear<'a,U,DI,C,NI,NO> +
          DeviceLinear<U,C,BC,PI,NI,NO,Output=<D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          Self: ForwardAll<Input=I> +
                PartialForward<DiffInput=DI,DiffOutput=<D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output> +
                PreTrain<U> {
    fn forward_diff(&self, input: Self::DiffInput) -> Result<Self::DiffOutput, EvaluateError> {
        Ok(self.device.forward_diff_linear(&self.units,input)?)
    }
}
impl<'a,U,C,BC,P,D,OP,I,DI,PI,const NI:usize,const NO:usize> ContinueForward for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: BackwardAll<U,LossInput=()> +
             ForwardAll<Input=I,Output=PI> + PartialForward<PartialOutput=PI> +
             PreTrain<U,PreOutput=PI> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          D: Device<U> + DeviceDiffLinear<'a,U,DI,C,NI,NO,Output=Self::PartialOutput> +
             DeviceLinear<U,C,BC,PI,NI,NO,Output=<D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          Self: ForwardAll<Input=I,Output=Self::PartialOutput> +
                PartialForward<DiffInput=DI,DiffOutput=<D as DeviceDiffLinear<'a,U,DI,C,NI,NO>>::Output> +
                PreTrain<U> {
    type ConinueOutput = Self::PartialOutput;
    fn continue_forward(&self, input: &Self::PartialOutput) -> Result<Self::ConinueOutput, EvaluateError> {
        Ok(self.device.clone_diff_linear_forward_output(input)?)
    }
}
impl<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> Loss<U> for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: PreTrain<U,PreOutput=PI> +
             ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          D: Device<U> + DeviceDiffLinear<'a,U,DI,C,NI,NO> + DeviceLinear<U,C,BC,PI,NI,NO> +
             DeviceBatchAveraging<C,U> + DeviceBatchAveraging<BC,U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          C: Debug,
          BC: Debug,
          OP: Optimizer<U,D>,
          BC: From<<D as DeviceLinear<U,C,BC,PI,NI,NO>>::Output>,
          Self: BackwardAll<U> {
}
// OnStep implementation for DiffLinearLayer
impl<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> OnStep for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + PreTrain<U> + Loss<U> + OnStep,
          U: Default + Clone + Copy + UnitValue<U>,
          D: Device<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug,
          OP: Optimizer<U,D> {
    fn on_step(&mut self, step: usize) -> Result<(), TrainingError> {
        self.unit_optimizer.on_step(step)?;
        self.bias_optimizer.on_step(step)?;
        Ok(self.parent.on_step(step)?)
    }
    
    fn on_frequently_step(&mut self, step: usize, frequently_step: usize) -> Result<(), TrainingError> {
        self.unit_optimizer.on_frequently_step(step,frequently_step)?;
        self.bias_optimizer.on_frequently_step(step,frequently_step)?;
        Ok(self.parent.on_frequently_step(step,frequently_step)?)
    }
}
impl<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> PersistProgress<TextFilePersistence,Specialized>
    for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> + PreTrain<U> + Loss<U> +
             PersistProgress<TextFilePersistence,Specialized>,
          U: Default + Clone + Copy + UnitValue<U> + FromStr,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          DI: Debug,
          OP: Optimizer<U,D> + Persistence<U,TextFilePersistence,Specialized>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO>,
          TextRecord: From<U>,
          ModelLoadError: From<<U as FromStr>::Err> {
    fn load_progress(&mut self, persistence: &mut TextFilePersistence) -> Result<(), TrainingError> {
        self.parent.load_progress(persistence)?;

        self.unit_optimizer.load(persistence)?;
        self.bias_optimizer.load(persistence)?;

        Ok(())
    }

    fn save_progress(&mut self, persistence: &mut TextFilePersistence) -> Result<(), PersistenceError> {
        self.parent.save_progress(persistence)?;

        persistence.write_layer_start();
        persistence.write_units_start();

        self.unit_optimizer.save(persistence)?;

        persistence.write_units_start();

        self.bias_optimizer.save(persistence)?;

        persistence.write_layer_end();

        Ok(())
    }
}
impl<'a,T,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize> PersistProgress<T,Linear>
    for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where T: LinearPersistence<U>,
          P: ForwardAll<Input=I,Output=PI> +
             BackwardAll<U,LossInput=()> + PreTrain<U> + Loss<U> +
             PersistProgress<T,Linear>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          PI: Debug + BatchDataType,
          DI: Debug,
          OP: Optimizer<U,D> + Persistence<U,T,Linear>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    fn load_progress(&mut self, persistence: &mut T) -> Result<(), TrainingError> {
        self.parent.load_progress(persistence)?;

        self.unit_optimizer.load(persistence)?;
        self.bias_optimizer.load(persistence)?;

        Ok(())
    }

    fn save_progress(&mut self, persistence: &mut T) -> Result<(), PersistenceError> {
        self.parent.save_progress(persistence)?;

        self.unit_optimizer.save(persistence)?;
        self.bias_optimizer.save(persistence)?;

        Ok(())
    }
}
/// Trait for DiffLinearLayer instance creation
pub trait DiffLinearLayerInstantiation<'a,U,C,BC,P,OP,D,I,DI,PI,const NI:usize,const NO:usize>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=()> +
             PreTrain<U> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          D: Device<U>,
          OP: Optimizer<U,D> {
    /// Create an instance of DiffLinearLayers
    /// # Arguments
    /// * `parent` - upper layer
    /// * `device` - Device object used for neural network computation
    /// * `ui` - Callback to generate weight of unit
    /// * `bi` - Callback to generate weight of bias
    /// * `b` - optimizer builder
    ///
    /// # Errors
    ///
    /// This function may return the following errors
    /// * [`LayerInstantiationError`]
    fn instantiation<UI: FnMut() -> U, BI: FnMut() -> U, B: OptimizerBuilder<U,D,Output=OP>>(parent:P,device:&D,ui: UI, bi: BI, b: &B)
        -> Result<DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>,LayerInstantiationError>;
}
impl<'a,U,C,BC,P,D,OP,I,DI,PI,const NI:usize,const NO:usize> DiffLinearLayerInstantiation<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    for DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>
    where P: ForwardAll<Input=I,Output=PI> + BackwardAll<U,LossInput=()> +
             PreTrain<U> + Loss<U>,
          U: Default + Clone + Copy + UnitValue<U>,
          I: Debug + Send + Sync,
          DI: Debug,
          PI: Debug + BatchDataType,
          OP: Optimizer<U,D>,
          D: Device<U> + DeviceLinear<U,C,BC,PI,NI,NO> {
    fn instantiation<UI: FnMut() -> U, BI: FnMut() -> U, B: OptimizerBuilder<U,D,Output=OP>>(parent: P, device:&D,ui: UI, bi: BI, b: &B)
        -> Result<DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>,LayerInstantiationError> {
        Ok(DiffLinearLayer::<'a,_,_,_,_,_,_,_,_,PI,NI,NO>::new(parent,device,ui,bi,b)?)
    }
}
/// Builder for DiffLinearLayer instance creation
pub struct DiffLinearLayerBuilder<const NI:usize,const NO:usize> {
    ni:PhantomData<[();NI]>,
    no:PhantomData<[();NO]>
}
impl<const NI:usize,const NO:usize> DiffLinearLayerBuilder<NI,NO> {
    /// Create an instance of DiffLinearLayerBuilder
    pub fn new() -> DiffLinearLayerBuilder<NI, NO> {
        DiffLinearLayerBuilder {
            ni: PhantomData::<[(); NI]>,
            no: PhantomData::<[(); NO]>
        }
    }
}
impl<const NI:usize,const NO:usize> DiffLinearLayerBuilder<NI,NO> {
    /// Create an instance of LinearLayers
    /// # Arguments
    /// * `parent` - upper layer
    /// * `device` - Device object used for neural network computation
    /// * `ui` - Callback to generate weight of unit
    /// * `bi` - Callback to generate weight of bias
    /// * `b` - optimizer builder
    ///
    /// # Errors
    ///
    /// This function may return the following errors
    /// * [`LayerInstantiationError`]
    pub fn build<'a,U,C,BC,P,OP,B,D,I,DI,PI>(&self,parent: P, device:&D, ui: impl FnMut() -> U, bi: impl FnMut() -> U, b: &B)
                 -> Result<DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>,LayerInstantiationError>
        where P: ForwardAll<Input=I,Output=PI> + PartialForward<DiffInput=DI> + ForwardDiff +
                 BackwardAll<U,LossInput=()> +
                 PreTrain<U> + Loss<U>,
              U: Default + Clone + Copy + UnitValue<U>,
              I: Debug + Send + Sync,
              DI: Debug,
              PI: Debug + BatchDataType,
              D: Device<U>,
              OP: Optimizer<U,D>,
              B: OptimizerBuilder<U,D,Output=OP>,
              DiffLinearLayer<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO>: DiffLinearLayerInstantiation<'a,U,C,BC,P,OP,D,I,DI,PI,NI,NO> {

        DiffLinearLayer::instantiation(parent,device,ui,bi,b)
    }
}