use std::io::{self, Write};
use sha1::{Digest as Sha1Digest, Sha1};
use sha1dc::Hasher as Sha1dcHasher;
use sha2::Sha256;
use crate::error::{Error, Result};
use crate::objects::{HashAlgo, ObjectId, ObjectKind};
mod parallel;
pub use parallel::{
hash_objects_parallel, index_parallelism_from_config, par_hash_with, parallel_hash_worthwhile,
try_par_hash_with, try_par_hash_with_force, ParallelHashError, Parallelism, PAR_HASH_MIN_ITEMS,
PAR_HASH_MIN_TOTAL_BYTES,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Backend {
X86ShaNi,
Aarch64Sha,
Portable,
}
impl Backend {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::X86ShaNi => "x86_sha_ni",
Self::Aarch64Sha => "aarch64_sha",
Self::Portable => "portable",
}
}
}
impl std::fmt::Display for Backend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[must_use]
pub fn backend(algo: HashAlgo) -> Backend {
let _ = algo;
#[cfg(target_arch = "x86_64")]
{
if std::arch::is_x86_feature_detected!("sha") {
Backend::X86ShaNi
} else {
Backend::Portable
}
}
#[cfg(target_arch = "aarch64")]
{
if std::arch::is_aarch64_feature_detected!("sha2") {
Backend::Aarch64Sha
} else {
Backend::Portable
}
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
{
Backend::Portable
}
}
const OBJECT_HEADER_BUF_LEN: usize = 6 + 1 + 20 + 1;
#[derive(Debug, Clone)]
pub enum ChecksumHasher {
Sha1(Sha1),
Sha256(Sha256),
}
impl ChecksumHasher {
#[must_use]
pub fn new(algo: HashAlgo) -> Self {
algo.checksum_hasher()
}
#[must_use]
pub fn algo(self: &ChecksumHasher) -> HashAlgo {
match self {
Self::Sha1(_) => HashAlgo::Sha1,
Self::Sha256(_) => HashAlgo::Sha256,
}
}
pub fn update(&mut self, data: &[u8]) {
match self {
Self::Sha1(h) => h.update(data),
Self::Sha256(h) => h.update(data),
}
}
#[must_use]
pub fn finalize(self) -> Vec<u8> {
match self {
Self::Sha1(h) => h.finalize().to_vec(),
Self::Sha256(h) => h.finalize().to_vec(),
}
}
#[must_use]
pub fn finalize_into(self, out: &mut [u8]) -> usize {
let digest = self.finalize();
let len = digest.len();
assert!(
out.len() >= len,
"checksum digest buffer too short: need {len} bytes, have {}",
out.len()
);
out[..len].copy_from_slice(&digest);
len
}
}
impl Write for ChecksumHasher {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.update(buf);
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[derive(Debug, Clone)]
pub enum ObjectHasher {
Sha1(Sha1dcHasher),
Sha256(Sha256),
}
impl ObjectHasher {
#[must_use]
pub fn new(algo: HashAlgo) -> Self {
algo.hasher()
}
#[must_use]
pub fn algo(self: &ObjectHasher) -> HashAlgo {
match self {
Self::Sha1(_) => HashAlgo::Sha1,
Self::Sha256(_) => HashAlgo::Sha256,
}
}
pub fn update(&mut self, data: &[u8]) {
match self {
Self::Sha1(h) => h.update(data),
Self::Sha256(h) => h.update(data),
}
}
#[must_use]
pub fn for_object(algo: HashAlgo, kind: ObjectKind, len: u64) -> Self {
let mut hasher = Self::new(algo);
let mut header = [0u8; OBJECT_HEADER_BUF_LEN];
let header_len = write_object_header(&mut header, kind, len);
hasher.update(&header[..header_len]);
hasher
}
pub fn finalize(self) -> Result<ObjectId> {
match self {
Self::Sha1(h) => match h.finalize() {
Ok(digest) => Ok(object_id_from_sha1_digest(&digest)),
Err(collision) => Err(collision_to_error(collision)),
},
Self::Sha256(h) => Ok(ObjectId::from_bytes(h.finalize().as_slice())
.unwrap_or_else(|_| unreachable!("SHA-256 digest is 32 bytes"))),
}
}
pub fn finalize_into(self, out: &mut [u8]) -> Result<usize> {
match self {
Self::Sha1(h) => {
let oid = match h.finalize() {
Ok(digest) => object_id_from_sha1_digest(&digest),
Err(collision) => return Err(collision_to_error(collision)),
};
let bytes = oid.as_bytes();
if out.len() < bytes.len() {
return Err(Error::InvalidObjectId(format!(
"digest buffer too short: need {} bytes, have {}",
bytes.len(),
out.len()
)));
}
out[..bytes.len()].copy_from_slice(bytes);
Ok(bytes.len())
}
Self::Sha256(h) => {
let digest = h.finalize();
let len = digest.len();
if out.len() < len {
return Err(Error::InvalidObjectId(format!(
"digest buffer too short: need {len} bytes, have {}",
out.len()
)));
}
out[..len].copy_from_slice(&digest);
Ok(len)
}
}
}
}
impl Write for ObjectHasher {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.update(buf);
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
impl HashAlgo {
#[must_use]
pub fn checksum_hasher(self) -> ChecksumHasher {
match self {
Self::Sha1 => ChecksumHasher::Sha1(Sha1::new()),
Self::Sha256 => ChecksumHasher::Sha256(Sha256::new()),
}
}
#[must_use]
pub fn checksum_digest(self, data: &[u8]) -> Vec<u8> {
let mut hasher = self.checksum_hasher();
hasher.update(data);
hasher.finalize()
}
#[must_use]
pub fn hasher(self) -> ObjectHasher {
match self {
Self::Sha1 => ObjectHasher::Sha1(Sha1dcHasher::new()),
Self::Sha256 => ObjectHasher::Sha256(Sha256::new()),
}
}
pub fn digest(self, data: &[u8]) -> Result<ObjectId> {
let mut hasher = self.hasher();
hasher.update(data);
hasher.finalize()
}
pub fn hash_object(self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
hash_object(self, kind, data)
}
}
pub fn hash_object(algo: HashAlgo, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
let len = u64::try_from(data.len()).unwrap_or(u64::MAX);
let mut hasher = ObjectHasher::for_object(algo, kind, len);
hasher.update(data);
hasher.finalize()
}
fn object_id_from_sha1_digest(digest: &sha1dc::Digest) -> ObjectId {
ObjectId::from_bytes(digest.as_ref())
.unwrap_or_else(|_| unreachable!("SHA-1 digest is 20 bytes"))
}
fn collision_to_error(collision: sha1dc::Collision) -> Error {
Error::HashCollision {
oid: object_id_from_sha1_digest(&collision.digest()),
}
}
pub struct HashingWriter<W: Write> {
inner: W,
hasher: ChecksumHasher,
}
impl<W: Write> HashingWriter<W> {
#[must_use]
pub fn new(algo: HashAlgo, inner: W) -> Self {
Self {
inner,
hasher: algo.checksum_hasher(),
}
}
#[must_use]
pub fn get_ref(&self) -> &W {
&self.inner
}
pub fn get_mut(&mut self) -> &mut W {
&mut self.inner
}
pub fn finish(mut self) -> io::Result<(W, ObjectId)> {
let digest_bytes = self.hasher.finalize();
self.inner.write_all(&digest_bytes)?;
let digest = ObjectId::from_bytes(&digest_bytes)
.unwrap_or_else(|_| unreachable!("checksum digest length matches object id for algo"));
Ok((self.inner, digest))
}
}
impl<W: Write> Write for HashingWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
if n > 0 {
self.hasher.update(&buf[..n]);
}
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TrailerMismatch {
pub expected: Vec<u8>,
pub found: Vec<u8>,
}
impl std::fmt::Display for TrailerMismatch {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"trailer checksum mismatch (expected {}, found {})",
hex::encode(&self.expected),
hex::encode(&self.found)
)
}
}
impl std::error::Error for TrailerMismatch {}
pub fn verify_trailer(algo: HashAlgo, bytes: &[u8]) -> std::result::Result<(), TrailerMismatch> {
let trailer_len = algo.len();
if bytes.len() < trailer_len {
return Err(TrailerMismatch {
expected: Vec::new(),
found: bytes.to_vec(),
});
}
let split = bytes.len() - trailer_len;
let (body, trailer) = bytes.split_at(split);
let expected = algo.checksum_digest(body);
if expected.as_slice() == trailer {
Ok(())
} else {
Err(TrailerMismatch {
expected,
found: trailer.to_vec(),
})
}
}
fn write_object_header(buf: &mut [u8], kind: ObjectKind, len: u64) -> usize {
let kind_bytes = kind.as_str().as_bytes();
let mut pos = 0;
buf[pos..pos + kind_bytes.len()].copy_from_slice(kind_bytes);
pos += kind_bytes.len();
buf[pos] = b' ';
pos += 1;
pos += write_decimal(&mut buf[pos..], len);
buf[pos] = 0;
pos + 1
}
fn write_decimal(out: &mut [u8], mut n: u64) -> usize {
if n == 0 {
out[0] = b'0';
return 1;
}
let mut tmp = [0u8; 20];
let mut digits = 0usize;
while n > 0 {
tmp[digits] = b'0' + (n % 10) as u8;
digits += 1;
n /= 10;
}
for i in 0..digits {
out[i] = tmp[digits - 1 - i];
}
digits
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
fn hex_oid(_algo: HashAlgo, hex: &str) -> ObjectId {
ObjectId::from_hex(hex).expect("test vector hex")
}
fn assert_digest(algo: HashAlgo, data: &[u8], expected_hex: &str) {
let got = algo.digest(data).expect("digest");
assert_eq!(got, hex_oid(algo, expected_hex), "digest mismatch");
}
fn assert_checksum_digest(algo: HashAlgo, data: &[u8], expected_hex: &str) {
let got = algo.checksum_digest(data);
assert_eq!(hex::encode(&got), expected_hex, "checksum digest mismatch");
}
#[test]
fn checksum_hasher_nist_vectors_sha1_sha256() {
assert_checksum_digest(
HashAlgo::Sha1,
b"",
"da39a3ee5e6b4b0d3255bfef95601890afd80709",
);
assert_checksum_digest(
HashAlgo::Sha1,
b"abc",
"a9993e364706816aba3e25717850c26c9cd0d89d",
);
assert_checksum_digest(
HashAlgo::Sha256,
b"",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
);
assert_checksum_digest(
HashAlgo::Sha256,
b"abc",
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
);
}
#[test]
fn checksum_hasher_matches_object_hasher_digest() {
let data: Vec<u8> = (0..512).map(|i| (i % 251) as u8).collect();
for algo in [HashAlgo::Sha1, HashAlgo::Sha256] {
let mut checksum = algo.checksum_hasher();
checksum.update(&data);
let checksum_bytes = checksum.finalize();
assert_eq!(
checksum_bytes,
algo.digest(&data).expect("digest").as_bytes()
);
}
}
#[test]
fn hashing_writer_uses_checksum_hasher() {
let body = b"streaming checksum body";
let mut buf = Vec::new();
{
let mut w = HashingWriter::new(HashAlgo::Sha256, Cursor::new(&mut buf));
w.write_all(body).unwrap();
let (_, digest) = w.finish().unwrap();
assert_eq!(
digest.as_bytes(),
HashAlgo::Sha256.checksum_digest(body).as_slice()
);
}
verify_trailer(HashAlgo::Sha256, &buf).expect("trailer valid");
}
#[test]
fn sha1_nist_vectors() {
assert_digest(
HashAlgo::Sha1,
b"",
"da39a3ee5e6b4b0d3255bfef95601890afd80709",
);
assert_digest(
HashAlgo::Sha1,
b"abc",
"a9993e364706816aba3e25717850c26c9cd0d89d",
);
assert_digest(
HashAlgo::Sha1,
b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
"84983e441c3bd26ebaae4aa1f95129e5e54670f1",
);
let million_a = vec![b'a'; 1_000_000];
assert_digest(
HashAlgo::Sha1,
&million_a,
"34aa973cd4c4daa4f61eeb2bdbad27316534016f",
);
}
#[test]
fn sha256_nist_vectors() {
assert_digest(
HashAlgo::Sha256,
b"",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
);
assert_digest(
HashAlgo::Sha256,
b"abc",
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
);
assert_digest(
HashAlgo::Sha256,
b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1",
);
let million_a = vec![b'a'; 1_000_000];
assert_digest(
HashAlgo::Sha256,
&million_a,
"cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0",
);
}
#[test]
fn incremental_matches_oneshot_random_splits() {
let data: Vec<u8> = (0..4096).map(|i| (i % 251) as u8).collect();
for algo in [HashAlgo::Sha1, HashAlgo::Sha256] {
let oneshot = algo.digest(&data).expect("digest");
let mut state = 0xdead_beef_u64;
let mut pos = 0usize;
let mut hasher = algo.hasher();
while pos < data.len() {
state = state
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
let remaining = data.len() - pos;
let chunk = if remaining == 1 {
1
} else {
(state as usize % remaining).max(1)
};
hasher.update(&data[pos..pos + chunk]);
pos += chunk;
}
assert_eq!(hasher.finalize().expect("finalize"), oneshot);
}
}
#[test]
fn object_header_encoding() {
let kinds = [
(ObjectKind::Blob, "blob"),
(ObjectKind::Tree, "tree"),
(ObjectKind::Commit, "commit"),
(ObjectKind::Tag, "tag"),
];
let lens = [0_u64, 1, 4_294_967_297];
let cases: Vec<(ObjectKind, u64, String)> = kinds
.iter()
.flat_map(|&(kind, name)| {
lens.iter()
.map(move |&len| (kind, len, format!("{name} {len}\0")))
})
.collect();
for (kind, len, expect) in &cases {
let mut buf = [0u8; OBJECT_HEADER_BUF_LEN];
let n = write_object_header(&mut buf, *kind, *len);
assert_eq!(
&buf[..n],
expect.as_bytes(),
"header for {kind:?} len {len}"
);
}
}
#[test]
fn hash_object_all_kinds() {
for kind in [
ObjectKind::Blob,
ObjectKind::Tree,
ObjectKind::Commit,
ObjectKind::Tag,
] {
for payload in [b"" as &[u8], b"x"] {
let mut hasher =
ObjectHasher::for_object(HashAlgo::Sha1, kind, payload.len() as u64);
hasher.update(payload);
assert_eq!(
hash_object(HashAlgo::Sha1, kind, payload).expect("hash"),
hasher.finalize().expect("finalize")
);
}
}
}
struct PartialWrite {
max: usize,
buf: Vec<u8>,
fail: bool,
}
impl Write for PartialWrite {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if self.fail {
return Err(io::Error::new(
io::ErrorKind::Other,
"simulated write failure",
));
}
let n = buf.len().min(self.max);
self.buf.extend_from_slice(&buf[..n]);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[test]
fn hashing_writer_partial_write_checksum() {
let inner = PartialWrite {
max: 2,
buf: Vec::new(),
fail: false,
};
let mut w = HashingWriter::new(HashAlgo::Sha1, inner);
let n = w.write(b"abcdef").expect("write");
assert_eq!(n, 2);
assert_eq!(w.get_ref().buf, b"ab");
let (_, digest) = w.finish().expect("finish");
assert_eq!(
digest.as_bytes(),
HashAlgo::Sha1.checksum_digest(b"ab").as_slice()
);
assert_ne!(
digest.as_bytes(),
HashAlgo::Sha1.checksum_digest(b"abcdef").as_slice()
);
}
#[test]
fn hashing_writer_write_error_does_not_update_checksum() {
let inner = PartialWrite {
max: 64,
buf: Vec::new(),
fail: true,
};
let mut w = HashingWriter::new(HashAlgo::Sha1, inner);
assert!(w.write(b"abcdef").is_err());
w.get_mut().fail = false;
w.write_all(b"ab").expect("write");
let (_, digest) = w.finish().expect("finish");
assert_eq!(
digest.as_bytes(),
HashAlgo::Sha1.checksum_digest(b"ab").as_slice()
);
}
#[test]
fn hashing_writer_trailer_matches_digest() {
let body = b"index payload bytes";
let mut buf = Vec::new();
{
let mut w = HashingWriter::new(HashAlgo::Sha1, Cursor::new(&mut buf));
w.write_all(body).unwrap();
let (_, digest) = w.finish().unwrap();
assert_eq!(
digest.as_bytes(),
HashAlgo::Sha1.checksum_digest(body).as_slice()
);
assert_eq!(&buf[body.len()..], digest.as_bytes());
}
verify_trailer(HashAlgo::Sha1, &buf).expect("trailer valid");
}
#[test]
fn verify_trailer_mismatch_is_typed() {
let body = b"payload";
let mut file = body.to_vec();
file.extend_from_slice(&[0u8; 20]);
let err = verify_trailer(HashAlgo::Sha1, &file).unwrap_err();
assert!(matches!(err, TrailerMismatch { .. }));
assert_ne!(err.expected, err.found);
}
#[test]
fn finalize_into_and_write_trait() {
let mut hasher = HashAlgo::Sha256.hasher();
hasher.write_all(b"via write trait").unwrap();
let mut out = [0u8; 32];
let n = hasher.finalize_into(&mut out).unwrap();
assert_eq!(n, 32);
assert_eq!(
HashAlgo::Sha256.digest(b"via write trait").expect("digest"),
ObjectId::from_bytes(&out).unwrap()
);
}
#[test]
fn backend_matches_cpu() {
let expected = {
#[cfg(target_arch = "x86_64")]
{
if std::arch::is_x86_feature_detected!("sha") {
Backend::X86ShaNi
} else {
Backend::Portable
}
}
#[cfg(target_arch = "aarch64")]
{
if std::arch::is_aarch64_feature_detected!("sha2") {
Backend::Aarch64Sha
} else {
Backend::Portable
}
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
{
Backend::Portable
}
};
assert_eq!(super::backend(HashAlgo::Sha1), expected);
}
#[test]
fn sha1_object_hasher_matches_sha1_crate_random() {
let mut state = 0x9e37_79b9_u64;
for len in [0_usize, 1, 63, 64, 65, 127, 128, 129, 512, 4096, 8192] {
let mut data = vec![0_u8; len];
for b in &mut data {
state = state.wrapping_mul(6364136223846793005).wrapping_add(1);
*b = (state >> 32) as u8;
}
let mut plain = Sha1::new();
plain.update(&data);
let expected = plain.finalize();
let mut hasher = ObjectHasher::new(HashAlgo::Sha1);
hasher.update(&data);
assert_eq!(
hasher.finalize().expect("finalize").as_bytes(),
expected.as_slice(),
"len {len}"
);
}
}
#[test]
fn finalize_detects_shattered_prefix() {
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data");
for name in ["shattered-1-prefix.bin", "shattered-2-prefix.bin"] {
let bytes = std::fs::read(dir.join(name)).expect("fixture");
let mut hasher = ObjectHasher::new(HashAlgo::Sha1);
hasher.update(&bytes);
let err = hasher.finalize().expect_err("SHAttered block must collide");
assert!(matches!(err, Error::HashCollision { .. }));
assert!(matches!(
HashAlgo::Sha1.digest(&bytes),
Err(Error::HashCollision { .. })
));
let mut plain = Sha1::new();
plain.update(format!("blob {}\0", bytes.len()).as_bytes());
plain.update(&bytes);
let oid = hash_object(HashAlgo::Sha1, ObjectKind::Blob, &bytes)
.expect("blob holding a SHAttered prefix is not a collision");
assert_eq!(oid.as_bytes(), plain.finalize().as_slice());
}
}
#[test]
fn sha256_finalize_never_collides() {
let data: Vec<u8> = (0..1024).map(|i| (i % 251) as u8).collect();
let mut hasher = ObjectHasher::new(HashAlgo::Sha256);
hasher.update(&data);
hasher
.finalize()
.expect("SHA-256 has no collision detection");
}
fn sha256_backend_matches_sha2_dispatcher() {
#[cfg(target_arch = "x86_64")]
{
let got = super::backend(HashAlgo::Sha256);
if std::arch::is_x86_feature_detected!("sha") {
assert_eq!(got, Backend::X86ShaNi);
} else {
assert_eq!(got, Backend::Portable);
}
}
}
}