1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0
//! SHA-512 compression with aws-lc's `sha512-x86_64` routines, imported by
//! `admin/import-aws-lc`: AVX on Intel CPUs that have it, and scalar code
//! elsewhere.
//!
//! The AVX routine is the faster on Intel's cores and the slower on AMD's,
//! so rind picks by vendor as aws-lc does. aws-lc-verification's SAW proof
//! covers the AVX routine, as aws-lc builds it; nothing verifies the scalar
//! one.
use std::sync::OnceLock;
use super::cpu::{HaveAvx, is_intel};
pub(crate) fn sha512_compress_blocks(state: &mut [u64; 8], blocks: &[u8]) {
debug_assert!(blocks.len().is_multiple_of(128));
// Both routines run their loop at least once, so zero blocks would read
// past `blocks`.
if blocks.is_empty() {
return;
}
let (data, num) = (blocks.as_ptr(), blocks.len() / 128);
// Picked once, rather than testing two features on every compression.
static AVX: OnceLock<Option<HaveAvx>> = OnceLock::new();
match AVX.get_or_init(|| HaveAvx::check().filter(|_| is_intel())) {
// SAFETY: `blocks` is `num` readable blocks, and the token proves AVX.
Some(_) => unsafe { sha512_block_data_order_avx(state, data, num) },
// SAFETY: `blocks` is `num` readable blocks.
None => unsafe { sha512_block_data_order_nohw(state, data, num) },
}
}
/// The round constants, each pair twice for the vector additions, then the
/// byte-swap mask, as the routines load them.
#[repr(C, align(64))]
struct Table([u64; 164]);
static K512: Table = Table(include!("aws_lc/sha512-x86_64.K512.rs"));
// Both routines take the System V ABI on every OS, since their text is aws-lc's
// ELF and Mach-O flavors, which read their arguments from its registers.
#[unsafe(naked)]
unsafe extern "sysv64" fn sha512_block_data_order_avx(
state: *mut [u64; 8],
data: *const u8,
blocks: usize,
) {
#[cfg(not(target_vendor = "apple"))]
core::arch::naked_asm!(
include_str!("aws_lc/sha512_block_data_order_avx.linux.s"),
K512 = sym K512,
options(att_syntax),
);
#[cfg(target_vendor = "apple")]
core::arch::naked_asm!(
include_str!("aws_lc/sha512_block_data_order_avx.apple.s"),
K512 = sym K512,
options(att_syntax),
);
}
#[unsafe(naked)]
unsafe extern "sysv64" fn sha512_block_data_order_nohw(
state: *mut [u64; 8],
data: *const u8,
blocks: usize,
) {
#[cfg(not(target_vendor = "apple"))]
core::arch::naked_asm!(
include_str!("aws_lc/sha512_block_data_order_nohw.linux.s"),
K512 = sym K512,
options(att_syntax),
);
#[cfg(target_vendor = "apple")]
core::arch::naked_asm!(
include_str!("aws_lc/sha512_block_data_order_nohw.apple.s"),
K512 = sym K512,
options(att_syntax),
);
}
#[cfg(test)]
#[cfg_attr(coverage_nightly, coverage(off))]
mod tests {
use super::*;
// Every other test sees only the routine the vendor picks, so the two are
// checked against each other here: odd and even block counts, with
// distinct bytes in every block.
#[test]
fn avx_matches_nohw() {
if HaveAvx::check().is_none() {
return;
}
let blocks: Vec<u8> = (0..5 * 128u32)
.map(|i| (i.wrapping_mul(0x9e3779b9) >> 24) as u8)
.collect();
for n in 1..=5 {
let mut want = [
0x6a09e667f3bcc908,
0xbb67ae8584caa73b,
0x3c6ef372fe94f82b,
0xa54ff53a5f1d36f1,
0x510e527fade682d1,
0x9b05688c2b3e6c1f,
0x1f83d9abfb41bd6b,
0x5be0cd19137e2179,
];
let mut got = want;
// SAFETY: `blocks` holds at least `n` blocks, and the token proves
// AVX.
unsafe {
sha512_block_data_order_nohw(&mut want, blocks.as_ptr(), n);
sha512_block_data_order_avx(&mut got, blocks.as_ptr(), n);
}
assert_eq!(got, want, "{n} blocks");
}
}
}