use hmac::{Hmac, Mac};
use md4::Md4;
use md5::{Digest, Md5};
type HmacMd5 = Hmac<Md5>;
pub const RC4_KEY_LEN: usize = 16;
pub const RC4_CONFOUNDER_LEN: usize = 8;
pub const RC4_CHECKSUM_LEN: usize = 16;
pub const SIG_HMAC_MD5: i32 = -138;
pub fn nt_hash(password: &str) -> [u8; RC4_KEY_LEN] {
let mut md4 = Md4::new();
for u in password.encode_utf16() {
md4.update(u.to_le_bytes());
}
md4.finalize().into()
}
fn hmac_md5(key: &[u8], data: &[u8]) -> [u8; 16] {
let mut m = <HmacMd5 as Mac>::new_from_slice(key).expect("HMAC accepts any key length");
m.update(data);
m.finalize().into_bytes().into()
}
pub fn rc4_apply(key: &[u8], data: &[u8]) -> Vec<u8> {
assert!(!key.is_empty(), "RC4 key must be non-empty");
let mut s: [u8; 256] = std::array::from_fn(|i| i as u8);
let mut j = 0usize;
for i in 0..256 {
j = (j + s[i] as usize + key[i % key.len()] as usize) & 0xff;
s.swap(i, j);
}
let (mut i, mut j) = (0usize, 0usize);
data.iter()
.map(|&b| {
i = (i + 1) & 0xff;
j = (j + s[i] as usize) & 0xff;
s.swap(i, j);
b ^ s[(s[i] as usize + s[j] as usize) & 0xff]
})
.collect()
}
fn usage_t(usage: i32) -> [u8; 4] {
let ms = match usage {
3 => 8, 9 => 8, 23 => 13, u => u,
};
(ms as u32).to_le_bytes()
}
pub fn encrypt(
key: &[u8],
usage: i32,
confounder: &[u8; RC4_CONFOUNDER_LEN],
plaintext: &[u8],
) -> Vec<u8> {
let ki = hmac_md5(key, &usage_t(usage));
let mut data = confounder.to_vec();
data.extend_from_slice(plaintext);
let cksum = hmac_md5(&ki, &data);
let ke = hmac_md5(&ki, &cksum);
let mut out = cksum.to_vec();
out.extend_from_slice(&rc4_apply(&ke, &data));
out
}
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum Rc4Error {
TooShort,
ChecksumMismatch,
}
impl core::fmt::Display for Rc4Error {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::TooShort => write!(f, "RC4-HMAC ciphertext too short"),
Self::ChecksumMismatch => write!(f, "RC4-HMAC checksum mismatch (wrong key)"),
}
}
}
impl std::error::Error for Rc4Error {}
pub fn decrypt(key: &[u8], usage: i32, ciphertext: &[u8]) -> Result<Vec<u8>, Rc4Error> {
if ciphertext.len() < RC4_CHECKSUM_LEN + RC4_CONFOUNDER_LEN {
return Err(Rc4Error::TooShort);
}
let (cksum, enc) = ciphertext.split_at(RC4_CHECKSUM_LEN);
let ki = hmac_md5(key, &usage_t(usage));
let ke = hmac_md5(&ki, cksum);
let data = rc4_apply(&ke, enc);
let recomputed = hmac_md5(&ki, &data);
let mut diff = 0u8;
for i in 0..RC4_CHECKSUM_LEN {
diff |= recomputed[i] ^ cksum[i];
}
if diff != 0 {
return Err(Rc4Error::ChecksumMismatch);
}
Ok(data[RC4_CONFOUNDER_LEN..].to_vec())
}
pub fn hmac_md5_checksum(key: &[u8], usage: i32, data: &[u8]) -> [u8; 16] {
let ksign = hmac_md5(key, b"signaturekey\0");
let mut md5 = Md5::new();
md5.update(usage.to_le_bytes());
md5.update(data);
let tmp = md5.finalize();
hmac_md5(&ksign, &tmp)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn nt_hash_known_vectors() {
assert_eq!(hex(&nt_hash("")), "31d6cfe0d16ae931b73c59d7e0c089c0");
assert_eq!(
hex(&nt_hash("password")),
"8846f7eaee8fb117ad06bdd830b7586c"
);
}
#[test]
fn rc4_known_stream() {
assert_eq!(hex(&rc4_apply(b"Key", b"Plaintext")), "bbf316e8d940af0ad3");
assert_eq!(hex(&rc4_apply(b"Wiki", b"pedia")), "1021bf0420");
}
#[test]
fn hmac_md5_rfc2202_case1() {
let key = [0x0bu8; 16];
assert_eq!(
hex(&hmac_md5(&key, b"Hi There")),
"9294727a3638bb1c13f48ef8158bfc9d"
);
}
#[test]
fn rc4_hmac_roundtrip_and_tamper() {
let key = nt_hash("Passw0rd!");
let conf = [0xa5u8; RC4_CONFOUNDER_LEN];
let pt = b"etype-23 encrypted part";
let ct = encrypt(&key, 3, &conf, pt);
assert_eq!(decrypt(&key, 3, &ct).unwrap(), pt);
let mut bad = ct.clone();
*bad.last_mut().unwrap() ^= 0x01;
assert_eq!(decrypt(&key, 3, &bad), Err(Rc4Error::ChecksumMismatch));
assert_eq!(decrypt(&key, 3, &[0u8; 10]), Err(Rc4Error::TooShort));
}
#[test]
fn rc4_hmac_matches_ms_pac_forge_reference() {
let key = nt_hash("Diff3renti@l");
for usage in [1i32, 2, 3, 4, 7, 9, 11, 23] {
for len in [0usize, 1, 8, 16, 17, 31, 64] {
let conf = [0x3cu8; RC4_CONFOUNDER_LEN];
let pt: Vec<u8> = (0..len).map(|i| (i as u8).wrapping_mul(7)).collect();
let ours = encrypt(&key, usage, &conf, &pt);
let theirs = ms_pac_forge::checksum::rc4_encrypt(&key, usage, &pt, Some(conf));
assert_eq!(
ours, theirs,
"usage={usage} len={len}: RC4-HMAC diverged from reference"
);
assert_eq!(decrypt(&key, usage, &theirs).unwrap(), pt);
}
}
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
}