block_graph/builtin/
soft.rs

1
2impl AI<(Vec<f32>,Vec<f32>),f32> for CrossEntropyLayer{
3	fn forward(&self,(_output,_target):(Vec<f32>,Vec<f32>))->f32{
4		//let t=self.temperature;
5		//-if t.i
6
7		//-new().fix_type::<Vec<f32>>().log_softmax().forward_fixed(output).iter().zip(target.iter()).map(|(o,t)|o*t).fold(0.0,|acc,x|acc+x)
8		todo!()
9	}
10}
11impl AI<(Vec<f32>,u32),f32> for CrossEntropyLayer{
12	fn forward(&self,(output,target):(Vec<f32>,u32))->f32{
13		let t=self.temperature;
14		-if t.is_nan(){output[target as usize].ln()}else{LogSoftmaxLayer::new(t).forward_fixed(output)[target as usize]}
15	}
16}
17impl AI<Vec<f32>,Vec<f32>> for AbnormalSoftmaxLayer{
18	fn forward(&self,input:Vec<f32>)->Vec<f32>{
19		let max=input.iter().fold(f32::NEG_INFINITY,|x,&y|if x<y{y}else{x});
20		input.into_iter().map(|x|if x==max{1.0}else{(x-max).exp()}).collect()
21	}
22}
23impl AI<Vec<f32>,Vec<f32>> for LogSoftmaxLayer{
24	fn forward(&self,input:Vec<f32>)->Vec<f32>{
25		let t=self.temperature.recip();
26		let mut sum=0.0;
27		input.iter().for_each(|x|sum+=(t*x).exp());
28		let r=sum.ln();
29		let output:Vec<f32>=input.into_iter().map(|x|t*x-r).collect();
30		output
31	}
32}
33impl AI<Vec<f32>,Vec<f32>> for SoftmaxLayer{
34	fn forward(&self,input:Vec<f32>)->Vec<f32>{
35		let t=self.temperature.recip();
36		if t.is_nan(){
37			let mut count=0;
38			let max=input.iter().fold(f32::NEG_INFINITY,|x,&y|if x<y{
39				count=0;
40				y
41			}else{
42				if x==y{count+=1}
43				x
44			});
45			let r=(count as f32).recip();
46			return input.into_iter().map(|x|if x==max{r}else{0.0}).collect();
47		}
48		let max=input.iter().fold(f32::NEG_INFINITY,|x,&y|if x<y{y}else{x});
49		let mut sum=0.0;
50		let intermediate:Vec<f32>=input.into_iter().map(|x|if x==max{1.0}else{((x-max)*t).exp()}).inspect(|y|sum+=y).collect();
51		let r=sum.recip();
52		let output:Vec<f32>=intermediate.into_iter().map(|y|r*y).collect();
53		output
54	}
55}
56impl Op for ChooseLayer{
57	type Output=u32;
58}
59impl Op for CrossEntropyLayer{
60	type Output=Vec<f32>;
61}
62impl<A:AI<X,Y>+Op<Output=Y>,T,X,Y,Z> AI<(X,T),Z> for CrossEntropy<A> where CrossEntropyLayer:AI<(Y,T),Z>{
63	fn forward(&self,(input,target):(X,T))->Z{self.layer.forward((self.inner.forward(input),target))}
64	fn forward_mut(&mut self,(input,target):(X,T))->Z{self.layer.forward_mut((self.inner.forward_mut(input),target))}
65}
66impl<A:Op<Output=Y>,Y> Op for Choose<A> where ChooseLayer:AI<Y,u32>{
67	type Output=u32;
68}
69impl<A:Op<Output=Y>,Y> Op for CrossEntropy<A> where CrossEntropyLayer:AI<(Y,Y),Vec<f32>>{
70	type Output=Vec<f32>;
71}
72
73/// declares layer and wrapper structs and implements accessor functions, decompose and op for reduction operations that have dim and temperature as configuration fields. ai will still have to be externally implemented for the layer stuct
74macro_rules! soft_like{
75	(@aiwrap $layer:ident,$wrap:ident)=>{
76		impl<A:AI<X,Y>+Op<Output=Y>,X,Y,Z> AI<X,Z> for $wrap<A> where $layer:AI<Y,Z>{
77			fn forward(&self,input:X)->Z{self.layer.forward(self.inner.forward(input))}
78			fn forward_mut(&mut self,input:X)->Z{self.layer.forward_mut(self.inner.forward_mut(input))}
79		}
80	};
81	(@declare $layer:ident,$wrap:ident)=>{
82		impl Default for $layer{
83			fn default()->Self{
84				Self{dim:-1,temperature:1.0}
85			}
86		}
87		#[derive(Clone,Copy,Debug,Deserialize,PartialEq,Serialize)]
88		/// layer to apply an operation
89		pub struct $layer{dim:i32,temperature:f32}
90		#[derive(Clone,Copy,Debug,Default,Deserialize,PartialEq,Serialize)]// TODO eq and hash that do something about the float
91		/// wrapper to apply an operation
92		pub struct $wrap<A>{inner:A,layer:$layer}
93	};
94	(@decompose $layer:ident,$wrap:ident)=>{
95		impl Decompose for $layer{
96			fn compose((dim,temperature):Self::Decomposition)->Self{
97				Self{dim,temperature}
98			}
99			fn decompose(self)->Self::Decomposition{(self.dim,self.temperature)}
100			fn decompose_cloned(&self)->Self::Decomposition{(self.dim,self.temperature)}
101			type Decomposition=(i32,f32);
102		}
103		impl<A:Decompose> Decompose for $wrap<A>{
104			fn compose((inner,layer):Self::Decomposition)->Self{
105				Self{inner:A::compose(inner),layer:$layer::compose(layer)}
106			}
107			fn decompose(self)->Self::Decomposition{(self.inner.decompose(),self.layer.decompose())}
108			fn decompose_cloned(&self)->Self::Decomposition{(self.inner.decompose_cloned(),self.layer.decompose_cloned())}
109			type Decomposition=(A::Decomposition,<$layer as Decompose>::Decomposition);
110		}
111	};
112	(@impl $layer:ident,$wrap:ident)=>{
113		impl $layer{
114			/// gets the dimension
115			pub fn get_dim(&self)->i32{self.dim}
116			/// gets the temperature
117			pub fn get_temperature(&self)->f32{self.temperature}
118			/// creates a new layer
119			pub fn new(temperature:f32)->Self{
120				Self{dim:-1,temperature}
121			}
122			/// sets the dimension
123			pub fn set_dim(&mut self,dim:i32){self.dim=dim}
124			/// sets the mismatch behavior. A temperature of NaN will make the non soft version if possible. A finite temperature will make the soft version
125			pub fn set_temperature(&mut self,temperature:f32){self.temperature=temperature}
126			/// sets the dimension
127			pub fn with_dim(mut self,dim:i32)->Self{
128				self.dim=dim;
129				self
130			}
131			/// sets the temperature. A temperature of NaN will make the non soft version if possible. A finite temperature will make the soft version
132			pub fn with_temperature(mut self,temperature:f32)->Self{
133				self.temperature=temperature;
134				self
135			}
136		}
137		impl<A:IntoSequence<M>,M:AI<M::Output,M::Output>+Op> IntoSequence<M> for $wrap<A> where $layer:Into<M>{
138			fn into_sequence(self)->Sequential<Vec<M>>{self.inner.into_sequence().with_next(self.layer)}
139		}
140		impl<A:UnwrapInner> UnwrapInner for $wrap<A>{
141			fn unwrap_inner(self)->A::Inner{self.into_inner().unwrap_inner()}
142			type Inner=A::Inner;
143		}
144		impl<A> $wrap<A>{
145			pub fn get_dim(&self)->i32{self.layer.dim}
146			/// gets the temperature
147			pub fn get_temperature(&self)->f32{self.layer.temperature}
148			/// references the inner value
149			pub fn inner(&self)->&A{&self.inner}
150			/// references the inner value
151			pub fn inner_mut(&mut self)->&mut A{&mut self.inner}
152			/// converts into the inner value
153			pub fn into_inner(self)->A{self.inner}
154			/// creates a new layer
155			pub fn new(inner:A,temperature:f32)->Self where Self:Op{
156				Self{inner,layer:$layer::new(temperature)}
157			}
158			/// sets the dimension
159			pub fn set_dim(&mut self,dim:i32){self.layer.dim=dim}
160			/// sets the temperature
161			pub fn set_temperature(&mut self,temperature:f32){self.layer.temperature=temperature}
162			/// sets the dimension
163			pub fn with_dim(mut self,dim:i32)->Self{
164				self.layer.dim=dim;
165				self
166			}
167			/// sets the inner module
168			pub fn with_inner<B>(self,inner:B)->$wrap<B> where $wrap<B>:Op{
169				$wrap{inner,layer:self.layer}
170			}
171			/// sets the mismatch behavior
172			pub fn with_temperature(mut self,temperature:f32)->Self{
173				self.layer.temperature=temperature;
174				self
175			}
176		}
177	};
178	(@op $layer:ident,$wrap:ident)=>{
179		impl Op for $layer{
180			type Output=Vec<f32>;
181		}
182		impl<A:Op<Output=Y>,Y> Op for $wrap<A> where $layer:AI<Y,Vec<f32>>{
183			type Output=Vec<f32>;
184		}
185	};
186	($layer:ident,$wrap:ident)=>{
187		soft_like!(@aiwrap @declare @decompose @impl @op $layer,$wrap);
188	};
189	($(@$command:tt)* $layer:ident,$wrap:ident)=>{
190		$(soft_like!(@$command $layer,$wrap);)*
191	};
192}
193soft_like!(@aiwrap @declare @decompose @impl ChooseLayer,Choose);
194soft_like!(AbnormalSoftmaxLayer,AbnormalSoftmax);
195soft_like!(SoftmaxLayer,Softmax);
196soft_like!(@declare @decompose @impl CrossEntropyLayer,CrossEntropy);
197soft_like!(LogSoftmaxLayer,LogSoftmax);
198use soft_like;
199use crate::{
200	AI,Decompose,IntoSequence,Op,UnwrapInner
201};
202use serde::{Deserialize,Serialize};
203use super::Sequential;
block_graph/builtin/soft.rs

block_graph/builtin/
soft.rs
