1#![allow(clippy::needless_range_loop)]
20
21use crate::aes::{Aes128, Aes192, Aes256, BLOCK_LEN};
22use crate::modes::increment_be32;
23use ic_core::traits::{Aead, Algorithm, BlockCipher, SelfTest};
24use ic_core::{ensure, Result, Zeroize};
25
26const R: u8 = 0xe1;
28
29pub struct GcmLimits;
33
34impl GcmLimits {
35 pub const MAX_PLAINTEXT_BYTES: u64 = (1 << 36) - 32;
37 pub const MAX_RANDOM_NONCE_INVOCATIONS: u64 = 1 << 32;
39 pub const RECOMMENDED_NONCE_LEN: usize = 12;
42}
43
44#[inline]
50#[must_use]
51pub fn ghash_accelerated() -> bool {
52 ic_core::cpu::has_ghash_clmul()
53}
54
55pub(crate) fn portable_ghash_mul(x: &mut [u8; BLOCK_LEN], h: &[u8; BLOCK_LEN]) {
60 let mut z = [0u8; BLOCK_LEN];
61 let mut v = *h;
62 for i in 0..128 {
63 let bit = (x[i / 8] >> (7 - (i % 8))) & 1;
64 let m = bit.wrapping_neg();
65 for j in 0..BLOCK_LEN {
66 z[j] ^= v[j] & m;
67 }
68 let lsb = v[BLOCK_LEN - 1] & 1;
70 let mut carry = 0u8;
71 for byte in v.iter_mut() {
72 let next = *byte & 1;
73 *byte = (*byte >> 1) | (carry << 7);
74 carry = next;
75 }
76 v[0] ^= R & lsb.wrapping_neg();
77 }
78 *x = z;
79 z.zeroize();
80 v.zeroize();
81}
82
83struct Ghash {
85 h: [u8; BLOCK_LEN],
86 powers: [[u8; BLOCK_LEN]; 8],
104 #[cfg(all(target_arch = "x86_64", feature = "std"))]
106 high_powers: bool,
107 acc: [u8; BLOCK_LEN],
108 #[cfg(all(target_arch = "x86_64", feature = "std"))]
114 accelerated: bool,
115}
116
117impl Ghash {
118 fn new(h: [u8; BLOCK_LEN]) -> Self {
119 let mut me = Self {
120 h,
121 powers: [[0u8; BLOCK_LEN]; 8],
122 #[cfg(all(target_arch = "x86_64", feature = "std"))]
123 high_powers: false,
124 acc: [0u8; BLOCK_LEN],
125 #[cfg(all(target_arch = "x86_64", feature = "std"))]
126 accelerated: ghash_accelerated(),
127 };
128 me.powers[0] = h;
131 for slot in 1..4 {
132 let mut p = me.powers[slot - 1];
133 me.mul_by(&mut p, &h);
134 me.powers[slot] = p;
135 }
136 me
137 }
138
139 #[inline]
141 fn mul_by(&self, x: &mut [u8; BLOCK_LEN], y: &[u8; BLOCK_LEN]) {
142 #[cfg(all(target_arch = "x86_64", feature = "std"))]
143 if self.accelerated {
144 unsafe { crate::clmul::mul(x, y) };
147 return;
148 }
149 portable_ghash_mul(x, y);
150 }
151
152 #[inline]
156 fn absorb4(&mut self, blocks: &[u8]) {
157 debug_assert_eq!(blocks.len(), BLOCK_LEN * 4);
158 let mut terms = [[0u8; BLOCK_LEN]; 4];
159 for (i, t) in terms.iter_mut().enumerate() {
160 t.copy_from_slice(&blocks[i * BLOCK_LEN..(i + 1) * BLOCK_LEN]);
161 }
162 for j in 0..BLOCK_LEN {
165 terms[0][j] ^= self.acc[j];
166 }
167 let multipliers = [
168 &self.powers[3],
169 &self.powers[2],
170 &self.powers[1],
171 &self.powers[0],
172 ];
173 for (t, m) in terms.iter_mut().zip(multipliers) {
174 self.mul_by(t, m);
175 }
176 self.acc = terms[0];
177 for t in &terms[1..] {
178 for j in 0..BLOCK_LEN {
179 self.acc[j] ^= t[j];
180 }
181 }
182 }
183
184 #[inline]
186 fn mul_acc(&mut self) {
187 #[cfg(all(target_arch = "x86_64", feature = "std"))]
188 if self.accelerated {
189 unsafe { crate::clmul::mul(&mut self.acc, &self.h) };
192 return;
193 }
194 portable_ghash_mul(&mut self.acc, &self.h);
195 }
196
197 fn update_padded(&mut self, mut data: &[u8]) {
199 #[cfg(all(target_arch = "x86_64", feature = "std"))]
202 if self.accelerated && data.len() >= BLOCK_LEN * 8 {
203 if !self.high_powers {
204 for slot in 4..8 {
205 let mut p = self.powers[slot - 1];
206 self.mul_by(&mut p, &self.h);
207 self.powers[slot] = p;
208 }
209 self.high_powers = true;
210 }
211 let whole = data.len() - data.len() % (BLOCK_LEN * 8);
212 unsafe { crate::clmul::absorb8(&mut self.acc, &self.powers, &data[..whole]) };
216 data = &data[whole..];
217 }
218 while data.len() >= BLOCK_LEN * 4 {
221 self.absorb4(&data[..BLOCK_LEN * 4]);
222 data = &data[BLOCK_LEN * 4..];
223 }
224 for chunk in data.chunks(BLOCK_LEN) {
225 let mut block = [0u8; BLOCK_LEN];
226 block[..chunk.len()].copy_from_slice(chunk);
227 for j in 0..BLOCK_LEN {
228 self.acc[j] ^= block[j];
229 }
230 self.mul_acc();
231 }
232 }
233
234 fn finalize(self) -> [u8; BLOCK_LEN] {
235 self.acc
236 }
237}
238
239impl Drop for Ghash {
240 fn drop(&mut self) {
241 self.h.zeroize();
242 self.acc.zeroize();
243 for p in self.powers.iter_mut() {
246 p.zeroize();
247 }
248 }
249}
250
251fn derive_j0(nonce: &[u8], h: &[u8; BLOCK_LEN]) -> [u8; BLOCK_LEN] {
253 if nonce.len() == 12 {
254 let mut j0 = [0u8; BLOCK_LEN];
255 j0[..12].copy_from_slice(nonce);
256 j0[15] = 1;
257 j0
258 } else {
259 let mut g = Ghash::new(*h);
260 g.update_padded(nonce);
261 let mut len_block = [0u8; BLOCK_LEN];
262 len_block[8..].copy_from_slice(&((nonce.len() as u64) * 8).to_be_bytes());
263 g.update_padded(&len_block);
264 g.finalize()
265 }
266}
267
268fn gcm_core<C: BlockCipher>(
270 cipher: &C,
271 nonce: &[u8],
272 aad: &[u8],
273 in_out: &mut [u8],
274 encrypting: bool,
275) -> Result<[u8; BLOCK_LEN]> {
276 ensure!(
277 !nonce.is_empty(),
278 InvalidParameter,
279 "gcm nonce must be non-empty"
280 );
281 ensure!(
282 in_out.len() as u64 <= GcmLimits::MAX_PLAINTEXT_BYTES,
283 CounterExhausted,
284 "gcm plaintext exceeds 2^39-256 bits"
285 );
286
287 let mut h = [0u8; BLOCK_LEN];
289 cipher.encrypt_block(&mut h)?;
290
291 let j0 = derive_j0(nonce, &h);
292
293 let mut g = Ghash::new(h);
296 g.update_padded(aad);
297 if !encrypting {
298 g.update_padded(in_out);
299 }
300
301 const CTR_BATCH: usize = 8;
304 let mut counter = j0;
305 increment_be32(&mut counter);
306 let mut keystream = [0u8; BLOCK_LEN * CTR_BATCH];
307 for chunk in in_out.chunks_mut(BLOCK_LEN * CTR_BATCH) {
308 let blocks = chunk.len().div_ceil(BLOCK_LEN);
309 for i in 0..blocks {
310 keystream[i * BLOCK_LEN..(i + 1) * BLOCK_LEN].copy_from_slice(&counter);
311 increment_be32(&mut counter);
312 }
313 cipher.encrypt_blocks(&mut keystream[..blocks * BLOCK_LEN])?;
314 for (d, k) in chunk.iter_mut().zip(keystream.iter()) {
315 *d ^= k;
316 }
317 }
318 keystream.zeroize();
319
320 if encrypting {
321 g.update_padded(in_out);
322 }
323
324 let mut len_block = [0u8; BLOCK_LEN];
325 len_block[..8].copy_from_slice(&((aad.len() as u64) * 8).to_be_bytes());
326 len_block[8..].copy_from_slice(&((in_out.len() as u64) * 8).to_be_bytes());
327 g.update_padded(&len_block);
328
329 let mut tag = g.finalize();
330 let mut ek_j0 = j0;
331 cipher.encrypt_block(&mut ek_j0)?;
332 for j in 0..BLOCK_LEN {
333 tag[j] ^= ek_j0[j];
334 }
335 ek_j0.zeroize();
336 h.zeroize();
337 Ok(tag)
338}
339
340macro_rules! aes_gcm {
341 ($name:ident, $inner:ty, $id:literal, $disp:literal, $keylen:literal) => {
342 #[doc = concat!("SP 800-38D ", $disp, ".")]
343 pub struct $name($inner);
344
345 impl Algorithm for $name {
346 const ID: &'static str = $id;
347 const NAME: &'static str = $disp;
348 }
349
350 impl Aead for $name {
351 const KEY_LEN: usize = $keylen;
352 const NONCE_LEN: usize = 12;
353 const TAG_LEN: usize = 16;
354
355 fn new(key: &[u8]) -> Result<Self> {
356 Ok(Self(<$inner as BlockCipher>::new(key)?))
357 }
358
359 fn seal_detached(
360 &self,
361 nonce: &[u8],
362 aad: &[u8],
363 in_out: &mut [u8],
364 tag: &mut [u8],
365 ) -> Result<()> {
366 ensure!(tag.len() == 16, InvalidLength, "gcm tag buffer");
367 let t = gcm_core(&self.0, nonce, aad, in_out, true)?;
368 tag.copy_from_slice(&t);
369 Ok(())
370 }
371
372 fn open_detached(
373 &self,
374 nonce: &[u8],
375 aad: &[u8],
376 in_out: &mut [u8],
377 tag: &[u8],
378 ) -> Result<()> {
379 ensure!(tag.len() == 16, InvalidLength, "gcm tag");
380 let expected = gcm_core(&self.0, nonce, aad, in_out, false)?;
381 if ic_core::ct::verify(&expected, tag) {
382 Ok(())
383 } else {
384 in_out.zeroize();
386 Err(ic_core::err!(AuthenticationFailed, $id))
387 }
388 }
389 }
390
391 impl SelfTest for $name {
392 fn self_test() -> Result<()> {
393 let key = [0u8; $keylen];
394 let nonce = [0u8; 12];
395 let c = <Self as Aead>::new(&key)?;
396 let mut buf = [0u8; 16];
397 let mut tag = [0u8; 16];
398 c.seal_detached(&nonce, &[], &mut buf, &mut tag)?;
399 c.open_detached(&nonce, &[], &mut buf, &tag)?;
400 ensure!(buf == [0u8; 16], SelfTestFailed, $id);
401 tag[0] ^= 1;
403 ensure!(
404 c.open_detached(&nonce, &[], &mut buf, &tag).is_err(),
405 SelfTestFailed,
406 $id
407 );
408 Ok(())
409 }
410 }
411 };
412}
413
414aes_gcm!(Aes128Gcm, Aes128, "aes-128-gcm", "AES-128-GCM", 16);
415aes_gcm!(Aes192Gcm, Aes192, "aes-192-gcm", "AES-192-GCM", 24);
416aes_gcm!(Aes256Gcm, Aes256, "aes-256-gcm", "AES-256-GCM", 32);
417
418#[cfg(test)]
419mod tests {
420 use super::*;
421
422 #[test]
431 fn the_batched_ghash_agrees_with_the_serial_one() {
432 let h = [
433 0x66, 0xe9, 0x4b, 0xd4, 0xef, 0x8a, 0x2c, 0x3b, 0x88, 0x4c, 0xfa, 0x59, 0xca, 0x34,
434 0x2b, 0x2e,
435 ];
436
437 let mut checked = 0;
438 for len in [
440 0usize, 1, 15, 16, 17, 31, 63, 64, 65, 79, 80, 127, 128, 129, 255, 256, 1024, 1025,
441 ] {
442 let data: std::vec::Vec<u8> = (0..len)
443 .map(|i| ((i as u64).wrapping_mul(0x9e37_79b9) >> 3) as u8)
444 .collect();
445
446 let mut batched = Ghash::new(h);
447 batched.update_padded(&data);
448
449 let mut serial = Ghash::new(h);
451 for chunk in data.chunks(BLOCK_LEN) {
452 let mut block = [0u8; BLOCK_LEN];
453 block[..chunk.len()].copy_from_slice(chunk);
454 for j in 0..BLOCK_LEN {
455 serial.acc[j] ^= block[j];
456 }
457 serial.mul_acc();
458 }
459
460 assert_eq!(
461 batched.acc, serial.acc,
462 "batched and serial GHASH disagree at {len} bytes"
463 );
464 checked += 1;
465 }
466 assert_eq!(checked, 18, "the comparison did not run");
467
468 for (prefix, len) in [(17usize, 128usize), (16, 2048), (5, 2048 + 48), (33, 1023)] {
472 let data: std::vec::Vec<u8> = (0..prefix + len)
473 .map(|i| ((i as u64).wrapping_mul(0x2545_f491_4f6c_dd1d) >> 11) as u8)
474 .collect();
475 let (head, tail) = data.split_at(prefix);
476 let mut batched = Ghash::new(h);
477 batched.update_padded(head);
478 batched.update_padded(tail);
479
480 let mut serial = Ghash::new(h);
481 for part in [head, tail] {
482 for chunk in part.chunks(BLOCK_LEN) {
483 let mut block = [0u8; BLOCK_LEN];
484 block[..chunk.len()].copy_from_slice(chunk);
485 for j in 0..BLOCK_LEN {
486 serial.acc[j] ^= block[j];
487 }
488 serial.mul_acc();
489 }
490 }
491 assert_eq!(batched.acc, serial.acc, "prefix {prefix}, then {len} bytes");
492
493 #[cfg(all(target_arch = "x86_64", feature = "std"))]
496 assert_eq!(batched.high_powers, batched.accelerated);
497 }
498
499 let long = std::vec![0xa5u8; BLOCK_LEN * 4];
502 let mut g = Ghash::new(h);
503 g.absorb4(&long);
504 let mut serial = Ghash::new(h);
505 for chunk in long.chunks(BLOCK_LEN) {
506 for j in 0..BLOCK_LEN {
507 serial.acc[j] ^= chunk[j];
508 }
509 serial.mul_acc();
510 }
511 assert_eq!(g.acc, serial.acc, "absorb4 alone disagrees with four steps");
512 }
513
514 #[test]
516 fn the_precomputed_powers_are_powers_of_h() {
517 let h = [0x3cu8; BLOCK_LEN];
518 let mut g = Ghash::new(h);
519 g.update_padded(&[0u8; BLOCK_LEN * 8]);
522 #[cfg(all(target_arch = "x86_64", feature = "std"))]
523 let built = if g.high_powers { 8 } else { 4 };
524 #[cfg(not(all(target_arch = "x86_64", feature = "std")))]
525 let built = 4;
526
527 let mut expect = h;
528 for (i, stored) in g.powers[..built].iter().enumerate() {
529 if i > 0 {
530 g.mul_by(&mut expect, &h);
531 }
532 assert_eq!(*stored, expect, "powers[{i}] is not H^{}", i + 1);
533 }
534 for i in 1..built {
536 assert_ne!(g.powers[i - 1], g.powers[i]);
537 }
538 }
539 use ic_core::codec::{hex, unhex};
540
541 fn check(key: &str, nonce: &str, pt: &str, aad: &str, ct: &str, tag: &str) {
543 let k = unhex(key).unwrap();
544 let mut buf = unhex(pt).unwrap();
545 let mut got_tag = [0u8; 16];
546 let n = unhex(nonce).unwrap();
547 let a = unhex(aad).unwrap();
548
549 match k.len() {
550 16 => {
551 let c = Aes128Gcm::new(&k).unwrap();
552 c.seal_detached(&n, &a, &mut buf, &mut got_tag).unwrap();
553 }
554 24 => {
555 let c = Aes192Gcm::new(&k).unwrap();
556 c.seal_detached(&n, &a, &mut buf, &mut got_tag).unwrap();
557 }
558 _ => {
559 let c = Aes256Gcm::new(&k).unwrap();
560 c.seal_detached(&n, &a, &mut buf, &mut got_tag).unwrap();
561 }
562 }
563 assert_eq!(hex(&buf), ct, "ciphertext");
564 assert_eq!(hex(&got_tag), tag, "tag");
565 }
566
567 #[test]
568 fn gcm_spec_case_1_empty() {
569 check(
570 "00000000000000000000000000000000",
571 "000000000000000000000000",
572 "",
573 "",
574 "",
575 "58e2fccefa7e3061367f1d57a4e7455a",
576 );
577 }
578
579 #[test]
580 fn gcm_spec_case_2_single_block() {
581 check(
582 "00000000000000000000000000000000",
583 "000000000000000000000000",
584 "00000000000000000000000000000000",
585 "",
586 "0388dace60b6a392f328c2b971b2fe78",
587 "ab6e47d42cec13bdf53a67b21257bddf",
588 );
589 }
590
591 #[test]
595 fn gcm_spec_case_3_multi_block() {
596 check(
597 "feffe9928665731c6d6a8f9467308308",
598 "cafebabefacedbaddecaf888",
599 "d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255",
600 "",
601 "42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091473f5985",
602 "4d5c2af327cd64a62cf35abd2ba6fab4",
603 );
604 }
605
606 #[test]
607 fn gcm_spec_case_4_with_aad() {
608 check(
609 "feffe9928665731c6d6a8f9467308308",
610 "cafebabefacedbaddecaf888",
611 "d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39",
612 "feedfacedeadbeeffeedfacedeadbeefabaddad2",
613 "42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091",
614 "5bc94fbc3221a5db94fae95ae7121a47",
615 );
616 }
617
618 #[test]
620 fn gcm_short_nonce_uses_ghash_j0() {
621 check(
622 "feffe9928665731c6d6a8f9467308308",
623 "cafebabefacedbad",
624 "d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39",
625 "feedfacedeadbeeffeedfacedeadbeefabaddad2",
626 "61353b4c2806934a777ff51fa22a4755699b2a714fcdc6f83766e5f97b6c742373806900e49f24b22b097544d4896b424989b5e1ebac0f07c23f4598",
627 "3612d2e79e3b0785561be14aaca2fccb",
628 );
629 }
630
631 #[test]
632 fn aes256_gcm_vector() {
633 check(
634 "feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308",
635 "cafebabefacedbaddecaf888",
636 "d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39",
637 "feedfacedeadbeeffeedfacedeadbeefabaddad2",
638 "522dc1f099567d07f47f37a32a84427d643a8cdcbfe5c0c97598a2bd2555d1aa8cb08e48590dbb3da7b08b1056828838c5f61e6393ba7a0abcc9f662",
639 "76fc6ece0f4e1768cddf8853bb2d551b",
640 );
641 }
642
643 #[test]
644 fn roundtrip_and_tamper_detection() {
645 let c = Aes256Gcm::new(&[7u8; 32]).unwrap();
646 let nonce = [9u8; 12];
647 let aad = b"header";
648 let plaintext = b"attack at dawn, bring the ontology";
649
650 let mut buf = plaintext.to_vec();
651 let mut tag = [0u8; 16];
652 c.seal_detached(&nonce, aad, &mut buf, &mut tag).unwrap();
653 assert_ne!(&buf[..], &plaintext[..]);
654
655 let mut ok = buf.clone();
656 c.open_detached(&nonce, aad, &mut ok, &tag).unwrap();
657 assert_eq!(&ok[..], &plaintext[..]);
658
659 let mut bad = buf.clone();
661 bad[0] ^= 1;
662 assert!(c.open_detached(&nonce, aad, &mut bad, &tag).is_err());
663 assert_eq!(
664 bad,
665 vec![0u8; bad.len()],
666 "plaintext must be wiped on failure"
667 );
668
669 let mut wrong_aad = buf.clone();
671 assert!(c
672 .open_detached(&nonce, b"other", &mut wrong_aad, &tag)
673 .is_err());
674
675 let mut wrong_nonce = buf.clone();
677 assert!(c
678 .open_detached(&[0u8; 12], aad, &mut wrong_nonce, &tag)
679 .is_err());
680 }
681
682 #[test]
683 fn self_tests_pass() {
684 Aes128Gcm::self_test().unwrap();
685 Aes192Gcm::self_test().unwrap();
686 Aes256Gcm::self_test().unwrap();
687 }
688}