block_graph/builtin/
soft.rs

1
2impl AI<(Vec<f32>,Vec<f32>),f32> for CrossEntropyLayer{
3	fn forward(&self,(_output,_target):(Vec<f32>,Vec<f32>))->f32{
4		//let t=self.temperature;
5		//-if t.i
6
7		//-new().fix_type::<Vec<f32>>().log_softmax().forward_fixed(output).iter().zip(target.iter()).map(|(o,t)|o*t).fold(0.0,|acc,x|acc+x)
8		todo!()
9	}
10}
11impl AI<(Vec<f32>,u32),f32> for CrossEntropyLayer{
12	fn forward(&self,(output,target):(Vec<f32>,u32))->f32{
13		let t=self.temperature;
14		-if t.is_nan(){output[target as usize].ln()}else{LogSoftmaxLayer::new(t).forward_fixed(output)[target as usize]}
15	}
16}
17impl AI<Vec<f32>,Vec<f32>> for AbnormalSoftmaxLayer{
18	fn forward(&self,input:Vec<f32>)->Vec<f32>{
19		let scale=1.0/self.temperature;
20
21		let max=input.iter().map(|x|x*scale).fold(f32::NEG_INFINITY,|x,y|if x<y{y}else{x});
22		input.into_iter().map(|x|x*scale).map(|x|if x==max{1.0}else{(x-max).exp()}).collect()
23	}
24}
25impl AI<Vec<f32>,Vec<f32>> for LogSoftmaxLayer{
26	fn forward(&self,input:Vec<f32>)->Vec<f32>{
27		let t=self.temperature.recip();
28		let mut sum=0.0;
29		input.iter().for_each(|x|sum+=(t*x).exp());
30		let r=sum.ln();
31		let output:Vec<f32>=input.into_iter().map(|x|t*x-r).collect();
32		output
33	}
34}
35impl AI<Vec<f32>,Vec<f32>> for SoftmaxLayer{
36	fn forward(&self,input:Vec<f32>)->Vec<f32>{
37		let t=self.temperature.recip();
38		if t.is_nan(){
39			let mut count=0;
40			let max=input.iter().fold(f32::NEG_INFINITY,|x,&y|if x<y{
41				count=0;
42				y
43			}else{
44				if x==y{count+=1}
45				x
46			});
47			let r=(count as f32).recip();
48			return input.into_iter().map(|x|if x==max{r}else{0.0}).collect();
49		}
50		let max=input.iter().fold(f32::NEG_INFINITY,|x,&y|if x<y{y}else{x});
51		let mut sum=0.0;
52		let intermediate:Vec<f32>=input.into_iter().map(|x|if x==max{1.0}else{((x-max)*t).exp()}).inspect(|y|sum+=y).collect();
53		let r=sum.recip();
54		let output:Vec<f32>=intermediate.into_iter().map(|y|r*y).collect();
55		output
56	}
57}
58impl Op for ChooseLayer{
59	type Output=u32;
60}
61impl Op for CrossEntropyLayer{
62	type Output=Vec<f32>;
63}
64impl<A:AI<X,Y>+Op<Output=Y>,T,X,Y,Z> AI<(X,T),Z> for CrossEntropy<A> where CrossEntropyLayer:AI<(Y,T),Z>{
65	fn forward(&self,(input,target):(X,T))->Z{self.layer.forward((self.inner.forward(input),target))}
66	fn forward_mut(&mut self,(input,target):(X,T))->Z{self.layer.forward_mut((self.inner.forward_mut(input),target))}
67}
68impl<A:Op<Output=Y>,Y> Op for Choose<A> where ChooseLayer:AI<Y,u32>{
69	type Output=u32;
70}
71impl<A:Op<Output=Y>,Y> Op for CrossEntropy<A> where CrossEntropyLayer:AI<(Y,Y),Vec<f32>>{
72	type Output=Vec<f32>;
73}
74
75/// declares layer and wrapper structs and implements accessor functions, decompose and op for reduction operations that have dim and temperature as configuration fields. ai will still have to be externally implemented for the layer stuct
76macro_rules! soft_like{
77	(@aiwrap $layer:ident,$wrap:ident)=>{
78		impl<A:AI<X,Y>+Op<Output=Y>,X,Y,Z> AI<X,Z> for $wrap<A> where $layer:AI<Y,Z>{
79			fn forward(&self,input:X)->Z{self.layer.forward(self.inner.forward(input))}
80			fn forward_mut(&mut self,input:X)->Z{self.layer.forward_mut(self.inner.forward_mut(input))}
81		}
82	};
83	(@declare $layer:ident,$wrap:ident)=>{
84		impl Default for $layer{
85			fn default()->Self{
86				Self{dim:-1,temperature:1.0}
87			}
88		}
89		#[derive(Clone,Copy,Debug,Deserialize,PartialEq,Serialize)]
90		/// layer to apply an operation
91		pub struct $layer{dim:i32,temperature:f32}
92		#[derive(Clone,Copy,Debug,Default,Deserialize,PartialEq,Serialize)]// TODO eq and hash that do something about the float
93		/// wrapper to apply an operation
94		pub struct $wrap<A>{inner:A,layer:$layer}
95	};
96	(@decompose $layer:ident,$wrap:ident)=>{
97		impl Decompose for $layer{
98			fn compose((dim,temperature):Self::Decomposition)->Self{
99				Self{dim,temperature}
100			}
101			fn decompose(self)->Self::Decomposition{(self.dim,self.temperature)}
102			fn decompose_cloned(&self)->Self::Decomposition{(self.dim,self.temperature)}
103			type Decomposition=(i32,f32);
104		}
105		impl<A:Decompose> Decompose for $wrap<A>{
106			fn compose((inner,layer):Self::Decomposition)->Self{
107				Self{inner:A::compose(inner),layer:$layer::compose(layer)}
108			}
109			fn decompose(self)->Self::Decomposition{(self.inner.decompose(),self.layer.decompose())}
110			fn decompose_cloned(&self)->Self::Decomposition{(self.inner.decompose_cloned(),self.layer.decompose_cloned())}
111			type Decomposition=(A::Decomposition,<$layer as Decompose>::Decomposition);
112		}
113	};
114	(@impl $layer:ident,$wrap:ident)=>{
115		impl $layer{
116			/// gets the dimension
117			pub fn get_dim(&self)->i32{self.dim}
118			/// gets the temperature
119			pub fn get_temperature(&self)->f32{self.temperature}
120			/// creates a new layer
121			pub fn new(temperature:f32)->Self{
122				Self{dim:-1,temperature}
123			}
124			/// sets the dimension
125			pub fn set_dim(&mut self,dim:i32){self.dim=dim}
126			/// sets the mismatch behavior. A temperature of NaN will make the non soft version if possible. A finite temperature will make the soft version
127			pub fn set_temperature(&mut self,temperature:f32){self.temperature=temperature}
128			/// sets the dimension
129			pub fn with_dim(mut self,dim:i32)->Self{
130				self.dim=dim;
131				self
132			}
133			/// sets the temperature. A temperature of NaN will make the non soft version if possible. A finite temperature will make the soft version
134			pub fn with_temperature(mut self,temperature:f32)->Self{
135				self.temperature=temperature;
136				self
137			}
138		}
139		impl<A:IntoSequence<M>,M:AI<M::Output,M::Output>+Op> IntoSequence<M> for $wrap<A> where $layer:Into<M>{
140			fn into_sequence(self)->Sequential<Vec<M>>{self.inner.into_sequence().with_next(self.layer)}
141		}
142		impl<A:UnwrapInner> UnwrapInner for $wrap<A>{
143			fn unwrap_inner(self)->A::Inner{self.into_inner().unwrap_inner()}
144			type Inner=A::Inner;
145		}
146		impl<A> $wrap<A>{
147			pub fn get_dim(&self)->i32{self.layer.dim}
148			/// gets the temperature
149			pub fn get_temperature(&self)->f32{self.layer.temperature}
150			/// references the inner value
151			pub fn inner(&self)->&A{&self.inner}
152			/// references the inner value
153			pub fn inner_mut(&mut self)->&mut A{&mut self.inner}
154			/// converts into the inner value
155			pub fn into_inner(self)->A{self.inner}
156			/// creates a new layer
157			pub fn new(inner:A,temperature:f32)->Self where Self:Op{
158				Self{inner,layer:$layer::new(temperature)}
159			}
160			/// sets the dimension
161			pub fn set_dim(&mut self,dim:i32){self.layer.dim=dim}
162			/// sets the temperature
163			pub fn set_temperature(&mut self,temperature:f32){self.layer.temperature=temperature}
164			/// sets the dimension
165			pub fn with_dim(mut self,dim:i32)->Self{
166				self.layer.dim=dim;
167				self
168			}
169			/// sets the inner module
170			pub fn with_inner<B>(self,inner:B)->$wrap<B> where $wrap<B>:Op{
171				$wrap{inner,layer:self.layer}
172			}
173			/// sets the mismatch behavior
174			pub fn with_temperature(mut self,temperature:f32)->Self{
175				self.layer.temperature=temperature;
176				self
177			}
178		}
179	};
180	(@op $layer:ident,$wrap:ident)=>{
181		impl Op for $layer{
182			type Output=Vec<f32>;
183		}
184		impl<A:Op<Output=Y>,Y> Op for $wrap<A> where $layer:AI<Y,Vec<f32>>{
185			type Output=Vec<f32>;
186		}
187	};
188	($layer:ident,$wrap:ident)=>{
189		soft_like!(@aiwrap @declare @decompose @impl @op $layer,$wrap);
190	};
191	($(@$command:tt)* $layer:ident,$wrap:ident)=>{
192		$(soft_like!(@$command $layer,$wrap);)*
193	};
194}
195soft_like!(@aiwrap @declare @decompose @impl ChooseLayer,Choose);
196soft_like!(AbnormalSoftmaxLayer,AbnormalSoftmax);
197soft_like!(SoftmaxLayer,Softmax);
198soft_like!(@declare @decompose @impl CrossEntropyLayer,CrossEntropy);
199soft_like!(LogSoftmaxLayer,LogSoftmax);
200use soft_like;
201use crate::{
202	AI,Decompose,IntoSequence,Op,UnwrapInner
203};
204use serde::{Deserialize,Serialize};
205use super::Sequential;
block_graph/builtin/soft.rs

block_graph/builtin/
soft.rs
