1use crate::grad_buffer::GradBuffer;
8use crate::node::BackwardNode;
9use crate::var::Var;
10use coeus_core::Scalar;
11use coeus_tensor::Tensor;
12use std::sync::Arc;
13
14pub trait AttentionMask: 'static + Copy + Clone + Send + Sync {
19 const IS_CAUSAL: bool;
21}
22
23#[derive(Debug, Clone, Copy, Default)]
25pub struct CausalMask;
26impl AttentionMask for CausalMask {
27 const IS_CAUSAL: bool = true;
28}
29
30#[derive(Debug, Clone, Copy, Default)]
32pub struct NullMask;
33impl AttentionMask for NullMask {
34 const IS_CAUSAL: bool = false;
35}
36
37pub struct ScaledDotProductAttnNode<
44 T: Scalar,
45 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
46 M: AttentionMask,
47> {
48 pub output_grad: Arc<GradBuffer<T, B>>,
50 pub inputs: Vec<Var<T, B>>,
52 pub q_clone: Tensor<T, B>,
54 pub k_clone: Tensor<T, B>,
56 pub v_clone: Tensor<T, B>,
58 pub attn_weights: Tensor<T, B>,
60 pub scale: T,
62 pub _mask: std::marker::PhantomData<M>,
64}
65
66impl<
67 T: coeus_ops::AttentionScalar,
68 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
69 M: AttentionMask,
70 > BackwardNode<T, B> for ScaledDotProductAttnNode<T, B, M>
71{
72 #[inline]
73 fn op_name(&self) -> &'static str {
74 "sdp_attention"
75 }
76
77 #[inline]
78 fn output_grad(&self) -> &Arc<GradBuffer<T, B>> {
79 &self.output_grad
80 }
81
82 #[inline]
83 fn inputs(&self) -> &[Var<T, B>] {
84 &self.inputs
85 }
86
87 fn backward(
88 &self,
89 grad_out: &Tensor<T, B>,
90 input_grads: &[Option<Arc<GradBuffer<T, B>>>],
91 ) -> Result<(), B::Error> {
92 let backend = B::default();
93
94 let need_gq = input_grads.first().and_then(|g| g.as_ref()).is_some();
95 let need_gk = input_grads.get(1).and_then(|g| g.as_ref()).is_some();
96 let need_gv = input_grads.get(2).and_then(|g| g.as_ref()).is_some();
97
98 if !need_gq && !need_gk && !need_gv {
99 return Ok(());
100 }
101
102 let mut grad_q = if need_gq {
103 Some(Tensor::zeros_on(self.q_clone.shape_cloned(), &backend)?)
104 } else {
105 None
106 };
107 let mut grad_k = if need_gk {
108 Some(Tensor::zeros_on(self.k_clone.shape_cloned(), &backend)?)
109 } else {
110 None
111 };
112 let mut grad_v = if need_gv {
113 Some(Tensor::zeros_on(self.v_clone.shape_cloned(), &backend)?)
114 } else {
115 None
116 };
117
118 coeus_ops::scaled_dot_product_attention_backward(
120 grad_out,
121 &self.q_clone,
122 &self.k_clone,
123 &self.v_clone,
124 &self.attn_weights,
125 self.scale,
126 grad_q.as_mut(),
127 grad_k.as_mut(),
128 grad_v.as_mut(),
129 &backend,
130 )?;
131
132 if let (Some(acc), Some(gq)) = (input_grads.first().and_then(|g| g.as_ref()), grad_q) {
133 let lock = acc.write();
134 coeus_ops::add_assign(lock, &gq, &backend)?;
135 }
136 if let (Some(acc), Some(gk)) = (input_grads.get(1).and_then(|g| g.as_ref()), grad_k) {
137 let lock = acc.write();
138 coeus_ops::add_assign(lock, &gk, &backend)?;
139 }
140 if let (Some(acc), Some(gv)) = (input_grads.get(2).and_then(|g| g.as_ref()), grad_v) {
141 let lock = acc.write();
142 coeus_ops::add_assign(lock, &gv, &backend)?;
143 }
144
145 Ok(())
146 }
147}
148
149type AttentionOutput<T, B> = (Var<T, B>, Tensor<T, B>);
154
155pub fn sdp_attention<
168 T: coeus_ops::AttentionScalar,
169 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
170 M: AttentionMask,
171>(
172 query: &Var<T, B>,
173 key: &Var<T, B>,
174 value: &Var<T, B>,
175 key_padding_mask: Option<&Var<T, B>>,
176 scale: T,
177) -> Result<AttentionOutput<T, B>, B::Error> {
178 let backend = B::default();
179
180 let (out_tensor, attn_weights) = coeus_ops::scaled_dot_product_attention(
181 &query.tensor,
182 &key.tensor,
183 &value.tensor,
184 key_padding_mask.map(|m| &m.tensor),
185 M::IS_CAUSAL,
186 scale,
187 &backend,
188 )?;
189
190 let requires_grad = crate::grad_mode::should_track_var(query)
191 || crate::grad_mode::should_track_var(key)
192 || crate::grad_mode::should_track_var(value);
193
194 if !requires_grad {
195 return Ok((Var::new(out_tensor, false)?, attn_weights));
196 }
197
198 let out_var = Var::from_tracked_op(out_tensor, requires_grad, &backend, |output_grad| {
199 ScaledDotProductAttnNode::<T, B, M> {
200 output_grad,
201 inputs: vec![query.clone(), key.clone(), value.clone()],
202 q_clone: query.tensor.clone(),
203 k_clone: key.tensor.clone(),
204 v_clone: value.tensor.clone(),
205 attn_weights: attn_weights.clone(),
206 scale,
207 _mask: std::marker::PhantomData,
208 }
209 })?;
210 Ok((out_var, attn_weights))
211}