1use crate::err::{Error, Result};
2use crate::nn::module::{Module, ShapeInfo, TrainMode};
3use crate::shapes::{DynShape, Shape};
4use crate::tensor::base::Tensor;
5use crate::tensor::device::Device;
6use crate::tensor::grad::RequiresGrad;
7use alloc::{
8 string::{String, ToString},
9 vec::Vec,
10};
11
12#[derive(Debug, Clone, Default)]
16pub struct ReLU;
17
18macro_rules! impl_stateless_shape_info {
19 ($($ty:ty),+ $(,)?) => {
20 $(impl ShapeInfo for $ty {
21 fn shape_info(&self) -> Option<String> {
22 None
23 }
24 })+
25 };
26}
27
28impl_stateless_shape_info!(ReLU);
29
30impl TrainMode for ReLU {}
34
35use crate::exec::catalog::op;
36use crate::tensor::backend::Execute;
37
38impl<
39 S: Shape + DynShape,
40 B: crate::tensor::backend::VariableBackend + Execute<op::Relu>,
41 G: RequiresGrad,
42> Module<Tensor<S, B, f32, G>> for ReLU
43where
44 <B as Execute<op::Relu>>::Output: Into<B::Storage<f32>>,
45{
46 type Output = Tensor<S, B, f32, G>;
48 type Error = Error;
50
51 #[inline]
52 fn forward(
54 &self,
55 x: Tensor<S, B, f32, G>,
56 ) -> core::result::Result<Tensor<S, B, f32, G>, Error> {
57 x.relu()
58 }
59}
60
61#[derive(Debug, Clone, Default)]
66pub struct GELU;
67
68impl_stateless_shape_info!(GELU);
69
70impl TrainMode for GELU {}
74
75impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Gelu>>
76 Module<Tensor<S, B>> for GELU
77where
78 <B as Execute<op::Gelu>>::Output: Into<B::Storage<f32>>,
79{
80 type Output = Tensor<S, B>;
82 type Error = Error;
84
85 #[inline]
86 fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
88 x.gelu()
89 }
90}
91
92#[derive(Debug, Clone, Default)]
97pub struct Swish;
98
99impl_stateless_shape_info!(Swish);
100
101impl TrainMode for Swish {}
105
106impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Swish>>
107 Module<Tensor<S, B>> for Swish
108where
109 <B as Execute<op::Swish>>::Output: Into<B::Storage<f32>>,
110{
111 type Output = Tensor<S, B>;
113 type Error = Error;
115
116 #[inline]
117 fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
119 x.swish()
120 }
121}
122
123#[derive(Debug, Clone, Default)]
128pub struct Mish;
129
130impl_stateless_shape_info!(Mish);
131
132impl TrainMode for Mish {}
136
137impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Mish>>
138 Module<Tensor<S, B>> for Mish
139where
140 <B as Execute<op::Mish>>::Output: Into<B::Storage<f32>>,
141{
142 type Output = Tensor<S, B>;
144 type Error = Error;
146
147 #[inline]
148 fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
150 x.mish()
151 }
152}
153
154#[derive(Debug, Clone, Default)]
160#[allow(clippy::upper_case_acronyms)]
161pub struct ELU;
162
163impl_stateless_shape_info!(ELU);
164
165impl TrainMode for ELU {}
169
170impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Elu>>
171 Module<Tensor<S, B>> for ELU
172where
173 <B as Execute<op::Elu>>::Output: Into<B::Storage<f32>>,
174{
175 type Output = Tensor<S, B>;
177 type Error = Error;
179
180 #[inline]
181 fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
183 x.elu()
184 }
185}
186
187#[derive(Debug, Clone)]
194pub struct Softmax {
195 pub dim: usize,
197}
198
199impl ShapeInfo for Softmax {
200 fn shape_info(&self) -> Option<String> {
201 None
202 }
203}
204
205impl Softmax {
206 pub fn new(dim: usize) -> Self {
208 Self { dim }
209 }
210}
211
212impl TrainMode for Softmax {}
216
217impl<
218 S: Shape + DynShape + crate::shapes::RuntimeRankProjection,
219 B: crate::tensor::backend::VariableBackend
220 + crate::tensor::backend::Execute<crate::exec::catalog::op::Softmax>,
221> Module<Tensor<S, B>> for Softmax
222where
223 <B as crate::tensor::backend::Execute<crate::exec::catalog::op::Softmax>>::Output:
224 Into<B::Storage<f32>>,
225{
226 type Output = Tensor<S, B>;
228 type Error = Error;
230
231 #[inline]
232 fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
234 x.softmax(self.dim as isize)
235 }
236}
237
238#[derive(Debug, Clone, Default)]
242pub struct Sigmoid;
243
244impl_stateless_shape_info!(Sigmoid);
245
246impl TrainMode for Sigmoid {}
250
251impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Sigmoid>>
252 Module<Tensor<S, B>> for Sigmoid
253where
254 <B as Execute<op::Sigmoid>>::Output: Into<B::Storage<f32>>,
255{
256 type Output = Tensor<S, B>;
258 type Error = Error;
260
261 #[inline]
262 fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
264 x.sigmoid()
265 }
266}
267
268#[derive(Debug, Clone, Default)]
272pub struct Tanh;
273
274impl_stateless_shape_info!(Tanh);
275
276impl TrainMode for Tanh {}
280
281impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Tanh>>
282 Module<Tensor<S, B>> for Tanh
283where
284 <B as Execute<op::Tanh>>::Output: Into<B::Storage<f32>>,
285{
286 type Output = Tensor<S, B>;
288 type Error = Error;
290
291 #[inline]
292 fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
294 x.tanh()
295 }
296}
297
298macro_rules! impl_stateless_state_visitors {
299 ($($t:ty),+ $(,)?) => {$ (
300 impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitState<B> for $t {
301 fn visit_state<V: crate::nn::StateVisitor<B>>(
302 &self,
303 _: &crate::nn::StatePath,
304 _: &mut V,
305 ) -> crate::err::Result<()> { Ok(()) }
306 }
307
308 impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitStateMut<B> for $t {
309 fn visit_state_mut<V: crate::nn::StateMutVisitor<B>>(
310 &mut self,
311 _: &crate::nn::StatePath,
312 _: &mut V,
313 ) -> crate::err::Result<()> { Ok(()) }
314 }
315
316 impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitParameters<B> for $t {
317 fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
318 &self,
319 _: &crate::nn::StatePath,
320 _: &mut V,
321 ) -> crate::err::Result<()> { Ok(()) }
322 }
323 )+ };
324}
325
326impl_stateless_state_visitors!(ReLU, GELU, Swish, Mish, ELU, Softmax, Sigmoid, Tanh);
327
328impl crate::nn::module::NamedLayers for ReLU {
329 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
331 vec![crate::nn::module::LayerNode {
332 name: prefix.to_string(),
333 type_name: alloc::string::String::from("ReLU"),
334 shape_info: "".to_string(),
335 children: vec![],
336 }]
337 }
338}
339
340impl crate::nn::module::NamedLayers for GELU {
341 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
343 vec![crate::nn::module::LayerNode {
344 name: prefix.to_string(),
345 type_name: alloc::string::String::from("GELU"),
346 shape_info: "".to_string(),
347 children: vec![],
348 }]
349 }
350}
351
352impl crate::nn::module::NamedLayers for Swish {
353 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
355 vec![crate::nn::module::LayerNode {
356 name: prefix.to_string(),
357 type_name: alloc::string::String::from("Swish"),
358 shape_info: "".to_string(),
359 children: vec![],
360 }]
361 }
362}
363
364impl crate::nn::module::NamedLayers for Sigmoid {
365 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
367 vec![crate::nn::module::LayerNode {
368 name: prefix.to_string(),
369 type_name: alloc::string::String::from("Sigmoid"),
370 shape_info: "".to_string(),
371 children: vec![],
372 }]
373 }
374}
375
376impl crate::nn::module::NamedLayers for Tanh {
377 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
379 vec![crate::nn::module::LayerNode {
380 name: prefix.to_string(),
381 type_name: alloc::string::String::from("Tanh"),
382 shape_info: "".to_string(),
383 children: vec![],
384 }]
385 }
386}
387
388impl crate::nn::module::NamedLayers for Softmax {
389 fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
391 vec![crate::nn::module::LayerNode {
392 name: prefix.to_string(),
393 type_name: alloc::string::String::from("Softmax"),
394 shape_info: format!("dim={}", self.dim),
395 children: vec![],
396 }]
397 }
398}
399
400macro_rules! impl_unit_to_device {
401 ($($t:ty),+) => {
402 $(
403 impl<B: crate::tensor::backend::VariableBackend, NewD: Device> crate::tensor::transfer::ToDevice<B, NewD> for $t {
404 type Output = $t;
405 fn to_device(self, _arg: &NewD::Arg) -> Result<Self::Output> {
406 Ok(self)
407 }
408 }
409 )+
410 };
411}
412
413impl_unit_to_device!(ReLU, GELU, Swish, Mish, ELU, Softmax, Sigmoid, Tanh);