1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0
//! AES-GCM on AVX and MOVBE without VAES, with aws-lc's `aesni_gcm_encrypt`
//! and `aesni_gcm_decrypt`, imported by `admin/import-aws-lc`, which
//! interleave AES-CTR and GHASH over six blocks at a time.
//!
//! aws-lc-verification's SAW proofs cover them as aws-lc builds them, for
//! AES-256 and whole blocks, not rind's copy. They support in-place use.
use core::arch::x86_64::__m128i;
use super::aes::HwKey;
use super::aes_gcm_vaes::add_to_counter;
use super::cpu::HaveAvxMovbeForAesGcm;
use super::ghash::{ClmulTable, Ghash};
/// The routines with the GHASH key powers they read.
pub(crate) struct AvxKernel {
htable: [__m128i; 9],
_token: HaveAvxMovbeForAesGcm,
}
impl AvxKernel {
/// The kernel for `table`'s GHASH key.
pub(crate) fn new(table: &ClmulTable, token: HaveAvxMovbeForAesGcm) -> Self {
Self {
htable: table.aws_lc_htable(),
_token: token,
}
}
/// Encrypts (if `ENC`) or decrypts a prefix of `inout` in place, from the
/// counter block after `y0`, folding the ciphertext into `ghash`, and
/// returns its length, a multiple of 96 bytes: zero below 288 bytes to
/// encrypt or 96 to decrypt.
pub(crate) fn cipher<const ENC: bool>(
&self,
key: &HwKey,
y0: &[u8; 16],
ghash: &mut Ghash<'_>,
inout: &mut [u8],
) -> usize {
let mut ivec = add_to_counter(y0, 1);
let mut xi = ghash.xi();
let (inout, len, key, htable) = (
inout.as_mut_ptr(),
inout.len(),
key.as_aes_key(),
&self.htable,
);
// SAFETY: `inout` is `len` readable and writable bytes, which the
// routines support in place, `key` is an `AES_KEY`, and `htable` the
// nine vectors they read. The token proves the CPU has AES-NI,
// PCLMULQDQ, AVX, and MOVBE.
let done = unsafe {
match ENC {
true => aesni_gcm_encrypt(inout, inout, len, key, &mut ivec, htable, &mut xi),
false => aesni_gcm_decrypt(inout, inout, len, key, &mut ivec, htable, &mut xi),
}
};
ghash.set_xi(&xi);
done
}
}
/// The routines' constants, which they load at offsets from the first.
#[repr(C, align(64))]
struct Table([u8; 144]);
static BSWAP_MASK: Table = Table(include!("aws_lc/aesni-gcm-x86_64.bswap_mask.rs"));
/// Declares a routine as a naked function over aws-lc's text, binding the
/// table and helpers through the `sym` operands listed. Each takes the System
/// V ABI on every OS, since its text is aws-lc's ELF and Mach-O flavors.
macro_rules! routine {
($name:ident($($arg:ident: $ty:ty),*) $(-> $ret:ty)? { $($op:ident = $sym:ident),* }) => {
#[unsafe(naked)]
unsafe extern "sysv64" fn $name($($arg: $ty),*) $(-> $ret)? {
#[cfg(not(target_vendor = "apple"))]
core::arch::naked_asm!(
include_str!(concat!("aws_lc/", stringify!($name), ".linux.s"))
$(, $op = sym $sym)*,
options(att_syntax),
);
#[cfg(target_vendor = "apple")]
core::arch::naked_asm!(
include_str!(concat!("aws_lc/", stringify!($name), ".apple.s"))
$(, $op = sym $sym)*,
options(att_syntax),
);
}
};
}
routine!(aesni_gcm_encrypt(
input: *const u8,
output: *mut u8,
len: usize,
key: *const u8,
ivec: *mut [u8; 16],
htable: *const [__m128i; 9],
xi: *mut [u8; 16]
) -> usize {
bswap_mask = BSWAP_MASK,
aesni_ctr32_6x = aesni_ctr32_6x,
aesni_ctr32_ghash_6x = aesni_ctr32_ghash_6x
});
routine!(aesni_gcm_decrypt(
input: *const u8,
output: *mut u8,
len: usize,
key: *const u8,
ivec: *mut [u8; 16],
htable: *const [__m128i; 9],
xi: *mut [u8; 16]
) -> usize {
bswap_mask = BSWAP_MASK,
aesni_ctr32_ghash_6x = aesni_ctr32_ghash_6x
});
// Helpers with aws-lc's own register convention, which only the routines
// above call.
routine!(aesni_ctr32_6x() {});
routine!(aesni_ctr32_ghash_6x() {});