oarust 0.8.6

GPU-first semantic computing engine written in Rust
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

// @generated by tool/gen/fn/generate.py; DO NOT EDIT.
// schema=ml_training.json schema_version=1 generator_version=64
// schema_sha256=6678961ee57f24310158676c7b4fe0821e21ca48db7edcb20d2a749a164976ae
// Exhaustive routing for schema-owned saved operation records.
use crate::{Matrix, Result};
use std::collections::HashMap;
#[path = "loss.gen.rs"]
mod loss;
pub(in crate::ml::autograd) use loss::{
	GradNodeBce, GradNodeCrossEntropy, GradNodeL1, GradNodeMaskedCrossEntropy, GradNodeMse,
	GradNodePpoClippedPolicy, GradNodeSmoothL1,
};
#[path = "matrix/activation.gen.rs"]
mod matrix_activation;
pub(in crate::ml::autograd) use matrix_activation::{
	GradNodeElu, GradNodeGelu, GradNodeLeakyRelu, GradNodeMish, GradNodeRelu, GradNodeSigmoid,
	GradNodeSilu, GradNodeSoftplus, GradNodeTanh,
};
#[path = "matrix/pool.gen.rs"]
mod matrix_pool;
pub(in crate::ml::autograd) use matrix_pool::{
	GradNodeAdaptiveAvgPool2d, GradNodeAvgPool2d, GradNodeMaxPool2d,
};
#[path = "matrix/rope.gen.rs"]
mod matrix_rope;
pub(in crate::ml::autograd) use matrix_rope::GradNodeRope;
#[path = "matrix/swiglu.gen.rs"]
mod matrix_swiglu;
pub(in crate::ml::autograd) use matrix_swiglu::{GradNodeSiluMul, GradNodeSwiglu};
#[path = "matrix/upsample.gen.rs"]
mod matrix_upsample;
pub(in crate::ml::autograd) use matrix_upsample::GradNodeUpsample2d;

pub(in crate::ml::autograd) enum BackwardStatus {
	Skipped,
	Recorded,
}

pub(in crate::ml::autograd) enum GradNodeOperation {
	PpoClippedPolicy(GradNodePpoClippedPolicy),
	CrossEntropy(GradNodeCrossEntropy),
	MaskedCrossEntropy(GradNodeMaskedCrossEntropy),
	SmoothL1(GradNodeSmoothL1),
	Mse(GradNodeMse),
	L1(GradNodeL1),
	Bce(GradNodeBce),
	Gelu(GradNodeGelu),
	Silu(GradNodeSilu),
	Relu(GradNodeRelu),
	Tanh(GradNodeTanh),
	Sigmoid(GradNodeSigmoid),
	LeakyRelu(GradNodeLeakyRelu),
	Elu(GradNodeElu),
	Mish(GradNodeMish),
	Softplus(GradNodeSoftplus),
	Swiglu(GradNodeSwiglu),
	Rope(GradNodeRope),
	AvgPool2d(GradNodeAvgPool2d),
	MaxPool2d(GradNodeMaxPool2d),
	AdaptiveAvgPool2d(GradNodeAdaptiveAvgPool2d),
	Upsample2d(GradNodeUpsample2d),
	SiluMul(GradNodeSiluMul),
}

impl GradNodeOperation {
	pub(in crate::ml::autograd) fn output_id(&self) -> u64 {
		match self {
			Self::PpoClippedPolicy(node) => node.output_id,
			Self::CrossEntropy(node) => node.output_id,
			Self::MaskedCrossEntropy(node) => node.output_id,
			Self::SmoothL1(node) => node.output_id,
			Self::Mse(node) => node.output_id,
			Self::L1(node) => node.output_id,
			Self::Bce(node) => node.output_id,
			Self::Gelu(node) => node.output_id,
			Self::Silu(node) => node.output_id,
			Self::Relu(node) => node.output_id,
			Self::Tanh(node) => node.output_id,
			Self::Sigmoid(node) => node.output_id,
			Self::LeakyRelu(node) => node.output_id,
			Self::Elu(node) => node.output_id,
			Self::Mish(node) => node.output_id,
			Self::Softplus(node) => node.output_id,
			Self::Swiglu(node) => node.output_id,
			Self::Rope(node) => node.output_id,
			Self::AvgPool2d(node) => node.output_id,
			Self::MaxPool2d(node) => node.output_id,
			Self::AdaptiveAvgPool2d(node) => node.output_id,
			Self::Upsample2d(node) => node.output_id,
			Self::SiluMul(node) => node.output_id,
		}
	}

	pub(in crate::ml::autograd) fn backward(
		&self,
		gradients: &mut HashMap<u64, Matrix>,
	) -> Result<BackwardStatus> {
		match self {
			Self::PpoClippedPolicy(node) => node.backward(gradients),
			Self::CrossEntropy(node) => node.backward(gradients),
			Self::MaskedCrossEntropy(node) => node.backward(gradients),
			Self::SmoothL1(node) => node.backward(gradients),
			Self::Mse(node) => node.backward(gradients),
			Self::L1(node) => node.backward(gradients),
			Self::Bce(node) => node.backward(gradients),
			Self::Gelu(node) => node.backward(gradients),
			Self::Silu(node) => node.backward(gradients),
			Self::Relu(node) => node.backward(gradients),
			Self::Tanh(node) => node.backward(gradients),
			Self::Sigmoid(node) => node.backward(gradients),
			Self::LeakyRelu(node) => node.backward(gradients),
			Self::Elu(node) => node.backward(gradients),
			Self::Mish(node) => node.backward(gradients),
			Self::Softplus(node) => node.backward(gradients),
			Self::Swiglu(node) => node.backward(gradients),
			Self::Rope(node) => node.backward(gradients),
			Self::AvgPool2d(node) => node.backward(gradients),
			Self::MaxPool2d(node) => node.backward(gradients),
			Self::AdaptiveAvgPool2d(node) => node.backward(gradients),
			Self::Upsample2d(node) => node.backward(gradients),
			Self::SiluMul(node) => node.backward(gradients),
		}
	}
}