use aes::cipher::{BlockDecrypt, BlockEncrypt, KeyInit, generic_array::GenericArray};
use aes::{Aes128, Aes256};
use md5::{Digest, Md5};
use sha2::{Sha256, Sha384, Sha512};
use crate::error::{PdfError, PdfResult};
use crate::types::{ObjectRef, PdfDictionary, PdfValue};
const PASSWORD_PADDING: [u8; 32] = [
0x28, 0xBF, 0x4E, 0x5E, 0x4E, 0x75, 0x8A, 0x41, 0x64, 0x00, 0x4E, 0x56, 0xFF, 0xFA, 0x01, 0x08,
0x2E, 0x2E, 0x00, 0xB6, 0xD0, 0x68, 0x3E, 0x80, 0x2F, 0x0C, 0xA9, 0xFE, 0x64, 0x53, 0x69, 0x7A,
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SecurityRevision {
R2,
R3,
R4,
R5,
R6,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CryptMethod {
Identity,
V2,
AesV2,
AesV3,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BytesKind {
String,
Stream,
}
#[derive(Debug, Clone)]
pub struct StandardSecurityHandler {
file_key: Vec<u8>,
string_method: CryptMethod,
stream_method: CryptMethod,
encrypt_metadata: bool,
}
impl StandardSecurityHandler {
pub fn open(
encrypt_dict: &PdfDictionary,
id_first: &[u8],
password: &[u8],
) -> PdfResult<Option<Self>> {
let filter = encrypt_dict
.get("Filter")
.and_then(PdfValue::as_name)
.unwrap_or("");
if filter != "Standard" {
return Err(PdfError::Unsupported(format!(
"encryption filter /{filter} is not supported"
)));
}
let v = encrypt_dict
.get("V")
.and_then(PdfValue::as_integer)
.unwrap_or(0);
let r = encrypt_dict
.get("R")
.and_then(PdfValue::as_integer)
.unwrap_or(0);
let revision = match r {
2 => SecurityRevision::R2,
3 => SecurityRevision::R3,
4 => SecurityRevision::R4,
5 => SecurityRevision::R5,
6 => SecurityRevision::R6,
other => {
return Err(PdfError::Unsupported(format!(
"Standard security handler revision {other} is not supported (only R=2..R=6 handled)"
)));
}
};
if v == 5 {
return open_v5(encrypt_dict, revision, password);
}
let (string_method, stream_method, key_length_bytes) = match v {
1 | 2 => {
let bits = encrypt_dict
.get("Length")
.and_then(PdfValue::as_integer)
.unwrap_or(40);
if bits % 8 != 0 || !(40..=128).contains(&bits) {
return Err(PdfError::Corrupt(format!(
"invalid /Length {bits} in Encrypt dictionary"
)));
}
(CryptMethod::V2, CryptMethod::V2, (bits / 8) as usize)
}
4 => {
let (strf, stmf) = resolve_v4_crypt_filters(encrypt_dict)?;
(strf, stmf, 16)
}
other => {
return Err(PdfError::Unsupported(format!(
"Standard security handler V={other} is not supported (only V=1, V=2, V=4, and V=5 handled)"
)));
}
};
let encrypt_metadata = if matches!(revision, SecurityRevision::R4) {
encrypt_dict
.get("EncryptMetadata")
.and_then(PdfValue::as_bool)
.unwrap_or(true)
} else {
true
};
let o = pdf_string_bytes(encrypt_dict, "O")?;
let u = pdf_string_bytes(encrypt_dict, "U")?;
let p = encrypt_dict
.get("P")
.and_then(PdfValue::as_integer)
.ok_or_else(|| PdfError::Corrupt("Encrypt dictionary missing /P".to_string()))?;
if o.len() != 32 || u.len() != 32 {
return Err(PdfError::Corrupt(
"Encrypt /O and /U must each be 32 bytes".to_string(),
));
}
let user_file_key = compute_file_key(
password,
&o,
p as i32,
id_first,
key_length_bytes,
revision,
encrypt_metadata,
);
if authenticate_user_password(&user_file_key, revision, &u, id_first) {
return Ok(Some(Self {
file_key: user_file_key,
string_method,
stream_method,
encrypt_metadata,
}));
}
let recovered_user_password =
recover_user_password_from_owner(password, &o, revision, key_length_bytes);
let owner_file_key = compute_file_key(
&recovered_user_password,
&o,
p as i32,
id_first,
key_length_bytes,
revision,
encrypt_metadata,
);
if authenticate_user_password(&owner_file_key, revision, &u, id_first) {
return Ok(Some(Self {
file_key: owner_file_key,
string_method,
stream_method,
encrypt_metadata,
}));
}
Ok(None)
}
pub fn encrypts_metadata(&self) -> bool {
self.encrypt_metadata
}
pub fn decrypt_bytes(
&self,
bytes: &[u8],
object_ref: ObjectRef,
kind: BytesKind,
) -> PdfResult<Vec<u8>> {
let method = match kind {
BytesKind::String => self.string_method,
BytesKind::Stream => self.stream_method,
};
match method {
CryptMethod::Identity => Ok(bytes.to_vec()),
CryptMethod::V2 => Ok(rc4(&self.object_key(object_ref, method), bytes)),
CryptMethod::AesV2 => aes_128_cbc_decrypt(&self.object_key(object_ref, method), bytes),
CryptMethod::AesV3 => {
aes_256_cbc_decrypt(&self.file_key, bytes)
}
}
}
fn object_key(&self, object_ref: ObjectRef, method: CryptMethod) -> Vec<u8> {
let suffix_len = if matches!(method, CryptMethod::AesV2) {
9
} else {
5
};
let mut material = Vec::with_capacity(self.file_key.len() + suffix_len);
material.extend_from_slice(&self.file_key);
let num = object_ref.object_number.to_le_bytes();
material.push(num[0]);
material.push(num[1]);
material.push(num[2]);
let generation = object_ref.generation.to_le_bytes();
material.push(generation[0]);
material.push(generation[1]);
if matches!(method, CryptMethod::AesV2) {
material.extend_from_slice(b"sAlT");
}
let digest = md5_bytes(&material);
let truncated_len = (self.file_key.len() + 5).min(16);
digest[..truncated_len].to_vec()
}
}
fn open_v5(
encrypt_dict: &PdfDictionary,
revision: SecurityRevision,
password: &[u8],
) -> PdfResult<Option<StandardSecurityHandler>> {
if !matches!(revision, SecurityRevision::R5 | SecurityRevision::R6) {
return Err(PdfError::Unsupported(format!(
"V=5 Encrypt dictionary requires R=5 or R=6, got {revision:?}"
)));
}
let (strf, stmf) = resolve_v5_crypt_filters(encrypt_dict)?;
let encrypt_metadata = encrypt_dict
.get("EncryptMetadata")
.and_then(PdfValue::as_bool)
.unwrap_or(true);
let o = pdf_string_bytes(encrypt_dict, "O")?;
let u = pdf_string_bytes(encrypt_dict, "U")?;
let oe = pdf_string_bytes(encrypt_dict, "OE")?;
let ue = pdf_string_bytes(encrypt_dict, "UE")?;
if o.len() != 48 || u.len() != 48 {
return Err(PdfError::Corrupt(
"V=5 Encrypt /O and /U must each be 48 bytes".to_string(),
));
}
if oe.len() != 32 || ue.len() != 32 {
return Err(PdfError::Corrupt(
"V=5 Encrypt /OE and /UE must each be 32 bytes".to_string(),
));
}
let truncated_password = &password[..password.len().min(127)];
let u_validation_salt = &u[32..40];
let u_key_salt = &u[40..48];
let user_hash = pdf_2_b_hash(truncated_password, u_validation_salt, None, revision);
if user_hash[..32] == u[..32] {
let intermediate_key = pdf_2_b_hash(truncated_password, u_key_salt, None, revision);
let file_key = aes_256_cbc_decrypt_no_pad(&intermediate_key[..32], &[0u8; 16], &ue)?;
return Ok(Some(StandardSecurityHandler {
file_key,
string_method: strf,
stream_method: stmf,
encrypt_metadata,
}));
}
let o_validation_salt = &o[32..40];
let o_key_salt = &o[40..48];
let owner_hash = pdf_2_b_hash(
truncated_password,
o_validation_salt,
Some(&u[..48]),
revision,
);
if owner_hash[..32] == o[..32] {
let intermediate_key =
pdf_2_b_hash(truncated_password, o_key_salt, Some(&u[..48]), revision);
let file_key = aes_256_cbc_decrypt_no_pad(&intermediate_key[..32], &[0u8; 16], &oe)?;
return Ok(Some(StandardSecurityHandler {
file_key,
string_method: strf,
stream_method: stmf,
encrypt_metadata,
}));
}
Ok(None)
}
fn pdf_2_b_hash(
password: &[u8],
salt: &[u8],
user_vector: Option<&[u8]>,
revision: SecurityRevision,
) -> Vec<u8> {
let mut hasher = Sha256::new();
hasher.update(password);
hasher.update(salt);
if let Some(vector) = user_vector {
hasher.update(vector);
}
let mut k: Vec<u8> = hasher.finalize().to_vec();
if matches!(revision, SecurityRevision::R5) {
return k;
}
let user_vector = user_vector.unwrap_or(&[]);
let mut round: u32 = 0;
loop {
let mut k1 = Vec::with_capacity((password.len() + k.len() + user_vector.len()) * 64);
for _ in 0..64 {
k1.extend_from_slice(password);
k1.extend_from_slice(&k);
k1.extend_from_slice(user_vector);
}
let key: [u8; 16] = k[..16].try_into().expect("K is at least 32 bytes");
let iv: [u8; 16] = k[16..32].try_into().expect("K is at least 32 bytes");
let encrypted = aes_128_cbc_encrypt_no_pad(&key, &iv, &k1);
let selector: u32 = encrypted[..16]
.iter()
.map(|byte| u32::from(*byte % 3))
.sum::<u32>()
% 3;
k = match selector {
0 => Sha256::digest(&encrypted).to_vec(),
1 => Sha384::digest(&encrypted).to_vec(),
_ => Sha512::digest(&encrypted).to_vec(),
};
let last_byte = *encrypted.last().expect("AES output is non-empty");
round += 1;
if round >= 64 && u32::from(last_byte) <= round.saturating_sub(32) {
break;
}
}
k.truncate(32);
k
}
fn resolve_v5_crypt_filters(encrypt_dict: &PdfDictionary) -> PdfResult<(CryptMethod, CryptMethod)> {
let strf = encrypt_dict
.get("StrF")
.and_then(PdfValue::as_name)
.unwrap_or("Identity");
let stmf = encrypt_dict
.get("StmF")
.and_then(PdfValue::as_name)
.unwrap_or("Identity");
let cf = encrypt_dict.get("CF").and_then(|value| match value {
PdfValue::Dictionary(dict) => Some(dict),
_ => None,
});
Ok((
resolve_crypt_filter_method(cf, strf)?,
resolve_crypt_filter_method(cf, stmf)?,
))
}
fn resolve_v4_crypt_filters(encrypt_dict: &PdfDictionary) -> PdfResult<(CryptMethod, CryptMethod)> {
let strf = encrypt_dict
.get("StrF")
.and_then(PdfValue::as_name)
.unwrap_or("Identity");
let stmf = encrypt_dict
.get("StmF")
.and_then(PdfValue::as_name)
.unwrap_or("Identity");
let cf = encrypt_dict.get("CF").and_then(|value| match value {
PdfValue::Dictionary(dict) => Some(dict),
_ => None,
});
Ok((
resolve_crypt_filter_method(cf, strf)?,
resolve_crypt_filter_method(cf, stmf)?,
))
}
fn resolve_crypt_filter_method(cf: Option<&PdfDictionary>, name: &str) -> PdfResult<CryptMethod> {
if name == "Identity" {
return Ok(CryptMethod::Identity);
}
let subfilter = cf
.and_then(|dict| dict.get(name))
.and_then(|value| match value {
PdfValue::Dictionary(dict) => Some(dict),
_ => None,
})
.ok_or_else(|| {
PdfError::Corrupt(format!(
"Encrypt /CF is missing the crypt filter entry /{name}"
))
})?;
let cfm = subfilter
.get("CFM")
.and_then(PdfValue::as_name)
.ok_or_else(|| {
PdfError::Corrupt(format!("crypt filter /{name} is missing the /CFM entry"))
})?;
match cfm {
"V2" => Ok(CryptMethod::V2),
"AESV2" => Ok(CryptMethod::AesV2),
"AESV3" => Ok(CryptMethod::AesV3),
"None" => Ok(CryptMethod::Identity),
other => Err(PdfError::Unsupported(format!(
"crypt filter method /{other} is not supported (only /V2, /AESV2, and /AESV3 handled)"
))),
}
}
fn aes_128_cbc_decrypt(key: &[u8], data: &[u8]) -> PdfResult<Vec<u8>> {
if key.len() != 16 {
return Err(PdfError::Corrupt(format!(
"AES-128 object key must be 16 bytes, got {}",
key.len()
)));
}
if data.len() < 32 || data.len() % 16 != 0 {
return Err(PdfError::Corrupt(format!(
"AES-128-CBC ciphertext must be at least 32 bytes and a multiple of 16; got {}",
data.len()
)));
}
let cipher = Aes128::new_from_slice(key)
.map_err(|error| PdfError::Corrupt(format!("AES-128 key rejected by cipher: {error}")))?;
let mut prev_block: [u8; 16] = data[..16].try_into().expect("slice is 16 bytes");
let mut output = Vec::with_capacity(data.len() - 16);
for chunk in data[16..].chunks(16) {
let mut block = GenericArray::clone_from_slice(chunk);
cipher.decrypt_block(&mut block);
for (plain_byte, iv_byte) in block.iter_mut().zip(prev_block.iter()) {
*plain_byte ^= iv_byte;
}
output.extend_from_slice(block.as_slice());
prev_block.copy_from_slice(chunk);
}
strip_pkcs7(output)
}
fn aes_256_cbc_decrypt(key: &[u8], data: &[u8]) -> PdfResult<Vec<u8>> {
if key.len() != 32 {
return Err(PdfError::Corrupt(format!(
"AES-256 file key must be 32 bytes, got {}",
key.len()
)));
}
if data.len() < 32 || data.len() % 16 != 0 {
return Err(PdfError::Corrupt(format!(
"AES-256-CBC ciphertext must be at least 32 bytes and a multiple of 16; got {}",
data.len()
)));
}
let cipher = Aes256::new_from_slice(key)
.map_err(|error| PdfError::Corrupt(format!("AES-256 key rejected by cipher: {error}")))?;
let mut prev_block: [u8; 16] = data[..16].try_into().expect("slice is 16 bytes");
let mut output = Vec::with_capacity(data.len() - 16);
for chunk in data[16..].chunks(16) {
let mut block = GenericArray::clone_from_slice(chunk);
cipher.decrypt_block(&mut block);
for (plain_byte, iv_byte) in block.iter_mut().zip(prev_block.iter()) {
*plain_byte ^= iv_byte;
}
output.extend_from_slice(block.as_slice());
prev_block.copy_from_slice(chunk);
}
strip_pkcs7(output)
}
fn aes_256_cbc_decrypt_no_pad(key: &[u8], iv: &[u8], data: &[u8]) -> PdfResult<Vec<u8>> {
if key.len() != 32 {
return Err(PdfError::Corrupt(format!(
"AES-256 key must be 32 bytes, got {}",
key.len()
)));
}
if iv.len() != 16 {
return Err(PdfError::Corrupt(format!(
"AES-256-CBC IV must be 16 bytes, got {}",
iv.len()
)));
}
if data.is_empty() || data.len() % 16 != 0 {
return Err(PdfError::Corrupt(format!(
"AES-256-CBC payload must be a non-empty multiple of 16 bytes; got {}",
data.len()
)));
}
let cipher = Aes256::new_from_slice(key)
.map_err(|error| PdfError::Corrupt(format!("AES-256 key rejected by cipher: {error}")))?;
let mut prev_block: [u8; 16] = iv.try_into().expect("iv length validated");
let mut output = Vec::with_capacity(data.len());
for chunk in data.chunks(16) {
let mut block = GenericArray::clone_from_slice(chunk);
cipher.decrypt_block(&mut block);
for (plain_byte, iv_byte) in block.iter_mut().zip(prev_block.iter()) {
*plain_byte ^= iv_byte;
}
output.extend_from_slice(block.as_slice());
prev_block.copy_from_slice(chunk);
}
Ok(output)
}
fn aes_128_cbc_encrypt_no_pad(key: &[u8; 16], iv: &[u8; 16], data: &[u8]) -> Vec<u8> {
assert!(
data.len() % 16 == 0,
"Algorithm 2.B K1 must be block-aligned, got {}",
data.len()
);
let cipher = Aes128::new_from_slice(key).expect("key length validated at compile time");
let mut output = Vec::with_capacity(data.len());
let mut prev: [u8; 16] = *iv;
for chunk in data.chunks(16) {
let mut buf = [0u8; 16];
for ((b, plain), iv_byte) in buf.iter_mut().zip(chunk.iter()).zip(prev.iter()) {
*b = plain ^ iv_byte;
}
let mut block = GenericArray::clone_from_slice(&buf);
cipher.encrypt_block(&mut block);
output.extend_from_slice(block.as_slice());
prev.copy_from_slice(block.as_slice());
}
output
}
fn strip_pkcs7(mut data: Vec<u8>) -> PdfResult<Vec<u8>> {
let Some(&pad) = data.last() else {
return Err(PdfError::Corrupt(
"AES-128-CBC plaintext is empty — missing PKCS#7 padding".to_string(),
));
};
if pad == 0 || pad > 16 || (pad as usize) > data.len() {
return Err(PdfError::Corrupt(format!(
"AES-128-CBC PKCS#7 padding byte {pad} is out of range"
)));
}
let new_len = data.len() - pad as usize;
if !data[new_len..].iter().all(|byte| *byte == pad) {
return Err(PdfError::Corrupt(
"AES-128-CBC PKCS#7 padding bytes do not match".to_string(),
));
}
data.truncate(new_len);
Ok(data)
}
fn pdf_string_bytes(dict: &PdfDictionary, key: &str) -> PdfResult<Vec<u8>> {
match dict.get(key) {
Some(PdfValue::String(s)) => Ok(s.0.clone()),
Some(_) => Err(PdfError::Corrupt(format!("Encrypt /{key} is not a string"))),
None => Err(PdfError::Corrupt(format!(
"Encrypt dictionary missing /{key}"
))),
}
}
fn compute_file_key(
password: &[u8],
o_entry: &[u8],
permissions: i32,
id_first: &[u8],
key_length_bytes: usize,
revision: SecurityRevision,
encrypt_metadata: bool,
) -> Vec<u8> {
let padded = pad_password(password);
let mut hasher = Md5::new();
hasher.update(padded);
hasher.update(o_entry);
hasher.update(permissions.to_le_bytes());
hasher.update(id_first);
if matches!(revision, SecurityRevision::R4) && !encrypt_metadata {
hasher.update([0xFFu8; 4]);
}
let mut digest = hasher.finalize_reset();
if matches!(revision, SecurityRevision::R3 | SecurityRevision::R4) {
for _ in 0..50 {
hasher.update(&digest[..key_length_bytes]);
digest = hasher.finalize_reset();
}
}
digest[..key_length_bytes].to_vec()
}
fn pad_password(password: &[u8]) -> [u8; 32] {
let mut out = [0u8; 32];
let take = password.len().min(32);
out[..take].copy_from_slice(&password[..take]);
if take < 32 {
out[take..].copy_from_slice(&PASSWORD_PADDING[..32 - take]);
}
out
}
fn recover_user_password_from_owner(
owner_password: &[u8],
o_entry: &[u8],
revision: SecurityRevision,
key_length_bytes: usize,
) -> Vec<u8> {
let padded = pad_password(owner_password);
let mut hasher = Md5::new();
hasher.update(padded);
let mut digest = hasher.finalize_reset();
if matches!(revision, SecurityRevision::R3 | SecurityRevision::R4) {
for _ in 0..50 {
hasher.update(&digest[..key_length_bytes]);
digest = hasher.finalize_reset();
}
}
let base_key = digest[..key_length_bytes].to_vec();
match revision {
SecurityRevision::R2 => rc4(&base_key, o_entry),
SecurityRevision::R3 | SecurityRevision::R4 => {
let mut buffer = o_entry.to_vec();
for i in (0u8..=19).rev() {
let key: Vec<u8> = base_key.iter().map(|byte| byte ^ i).collect();
buffer = rc4(&key, &buffer);
}
buffer
}
SecurityRevision::R5 | SecurityRevision::R6 => {
unreachable!("V=5 takes open_v5; Algorithm 7 is not applicable to R=5 / R=6")
}
}
}
fn authenticate_user_password(
file_key: &[u8],
revision: SecurityRevision,
u_entry: &[u8],
id_first: &[u8],
) -> bool {
match revision {
SecurityRevision::R2 => {
let encrypted = rc4(file_key, &PASSWORD_PADDING);
encrypted == u_entry
}
SecurityRevision::R5 | SecurityRevision::R6 => {
unreachable!("V=5 takes open_v5; Algorithm 5 is not applicable to R=5 / R=6")
}
SecurityRevision::R3 | SecurityRevision::R4 => {
let mut hasher = Md5::new();
hasher.update(PASSWORD_PADDING);
hasher.update(id_first);
let seed = hasher.finalize();
let mut buffer = rc4(file_key, &seed);
for i in 1u8..=19 {
let key: Vec<u8> = file_key.iter().map(|byte| byte ^ i).collect();
buffer = rc4(&key, &buffer);
}
buffer.as_slice() == &u_entry[..16]
}
}
}
fn md5_bytes(input: &[u8]) -> [u8; 16] {
let mut hasher = Md5::new();
hasher.update(input);
hasher.finalize().into()
}
fn rc4(key: &[u8], data: &[u8]) -> Vec<u8> {
let mut s: [u8; 256] = [0; 256];
for (index, value) in s.iter_mut().enumerate() {
*value = index as u8;
}
let mut j: u8 = 0;
for i in 0..256 {
j = j.wrapping_add(s[i]).wrapping_add(key[i % key.len()]);
s.swap(i, j as usize);
}
let mut output = Vec::with_capacity(data.len());
let mut i: u8 = 0;
let mut j: u8 = 0;
for &byte in data {
i = i.wrapping_add(1);
j = j.wrapping_add(s[i as usize]);
s.swap(i as usize, j as usize);
let k = s[(s[i as usize].wrapping_add(s[j as usize])) as usize];
output.push(byte ^ k);
}
output
}
#[cfg(test)]
pub(crate) mod test_helpers {
use super::*;
pub fn rc4(key: &[u8], data: &[u8]) -> Vec<u8> {
super::rc4(key, data)
}
pub fn compute_file_key(
password: &[u8],
o_entry: &[u8],
permissions: i32,
id_first: &[u8],
key_length_bytes: usize,
) -> Vec<u8> {
super::compute_file_key(
password,
o_entry,
permissions,
id_first,
key_length_bytes,
SecurityRevision::R3,
true,
)
}
pub fn compute_file_key_with_revision(
password: &[u8],
o_entry: &[u8],
permissions: i32,
id_first: &[u8],
key_length_bytes: usize,
revision: SecurityRevision,
) -> Vec<u8> {
super::compute_file_key(
password,
o_entry,
permissions,
id_first,
key_length_bytes,
revision,
true,
)
}
pub fn compute_file_key_r4(
password: &[u8],
o_entry: &[u8],
permissions: i32,
id_first: &[u8],
encrypt_metadata: bool,
) -> Vec<u8> {
super::compute_file_key(
password,
o_entry,
permissions,
id_first,
16,
SecurityRevision::R4,
encrypt_metadata,
)
}
pub fn compute_u_r3(file_key: &[u8], id_first: &[u8]) -> Vec<u8> {
let mut hasher = Md5::new();
hasher.update(PASSWORD_PADDING);
hasher.update(id_first);
let seed = hasher.finalize();
let mut buffer = super::rc4(file_key, &seed);
for i in 1u8..=19 {
let key: Vec<u8> = file_key.iter().map(|byte| byte ^ i).collect();
buffer = super::rc4(&key, &buffer);
}
buffer.resize(32, 0);
buffer
}
pub fn compute_o(
owner_password: &[u8],
user_password: &[u8],
revision: SecurityRevision,
key_length_bytes: usize,
) -> Vec<u8> {
let padded_owner = pad_password(owner_password);
let mut hasher = Md5::new();
hasher.update(padded_owner);
let mut digest = hasher.finalize_reset();
if matches!(revision, SecurityRevision::R3 | SecurityRevision::R4) {
for _ in 0..50 {
hasher.update(&digest[..key_length_bytes]);
digest = hasher.finalize_reset();
}
}
let base_key = digest[..key_length_bytes].to_vec();
let padded_user = pad_password(user_password);
match revision {
SecurityRevision::R2 => super::rc4(&base_key, &padded_user),
SecurityRevision::R3 | SecurityRevision::R4 => {
let mut buffer = super::rc4(&base_key, &padded_user);
for i in 1u8..=19 {
let key: Vec<u8> = base_key.iter().map(|byte| byte ^ i).collect();
buffer = super::rc4(&key, &buffer);
}
buffer
}
SecurityRevision::R5 | SecurityRevision::R6 => {
panic!("compute_o is not applicable to V=5 — use compute_v5_u / compute_v5_o")
}
}
}
pub fn object_key(file_key: &[u8], object_number: u32, generation: u16) -> Vec<u8> {
let mut material = Vec::with_capacity(file_key.len() + 5);
material.extend_from_slice(file_key);
let num = object_number.to_le_bytes();
material.push(num[0]);
material.push(num[1]);
material.push(num[2]);
let gen_bytes = generation.to_le_bytes();
material.push(gen_bytes[0]);
material.push(gen_bytes[1]);
let digest = super::md5_bytes(&material);
let truncated_len = (file_key.len() + 5).min(16);
digest[..truncated_len].to_vec()
}
pub fn object_key_aes(file_key: &[u8], object_number: u32, generation: u16) -> Vec<u8> {
let mut material = Vec::with_capacity(file_key.len() + 9);
material.extend_from_slice(file_key);
let num = object_number.to_le_bytes();
material.push(num[0]);
material.push(num[1]);
material.push(num[2]);
let gen_bytes = generation.to_le_bytes();
material.push(gen_bytes[0]);
material.push(gen_bytes[1]);
material.extend_from_slice(b"sAlT");
let digest = super::md5_bytes(&material);
let truncated_len = (file_key.len() + 5).min(16);
digest[..truncated_len].to_vec()
}
pub fn compute_v5_u_and_ue(
user_password: &[u8],
validation_salt: &[u8; 8],
key_salt: &[u8; 8],
file_key: &[u8; 32],
revision: SecurityRevision,
) -> (Vec<u8>, Vec<u8>) {
let verifier = super::pdf_2_b_hash(user_password, validation_salt, None, revision);
let mut u = Vec::with_capacity(48);
u.extend_from_slice(&verifier[..32]);
u.extend_from_slice(validation_salt);
u.extend_from_slice(key_salt);
let intermediate = super::pdf_2_b_hash(user_password, key_salt, None, revision);
let ue = aes_256_cbc_encrypt_no_pad(&intermediate[..32], &[0u8; 16], file_key);
(u, ue)
}
pub fn compute_v5_o_and_oe(
owner_password: &[u8],
validation_salt: &[u8; 8],
key_salt: &[u8; 8],
u_vector: &[u8; 48],
file_key: &[u8; 32],
revision: SecurityRevision,
) -> (Vec<u8>, Vec<u8>) {
let verifier =
super::pdf_2_b_hash(owner_password, validation_salt, Some(u_vector), revision);
let mut o = Vec::with_capacity(48);
o.extend_from_slice(&verifier[..32]);
o.extend_from_slice(validation_salt);
o.extend_from_slice(key_salt);
let intermediate = super::pdf_2_b_hash(owner_password, key_salt, Some(u_vector), revision);
let oe = aes_256_cbc_encrypt_no_pad(&intermediate[..32], &[0u8; 16], file_key);
(o, oe)
}
pub fn aes_256_cbc_encrypt(key: &[u8], iv: &[u8; 16], plaintext: &[u8]) -> Vec<u8> {
assert_eq!(key.len(), 32, "AES-256 key must be 32 bytes");
let cipher = Aes256::new_from_slice(key).expect("key length validated");
let pad_len = 16 - (plaintext.len() % 16);
let mut padded = Vec::with_capacity(plaintext.len() + pad_len);
padded.extend_from_slice(plaintext);
padded.extend(std::iter::repeat_n(pad_len as u8, pad_len));
let mut output = Vec::with_capacity(16 + padded.len());
output.extend_from_slice(iv);
let mut prev: [u8; 16] = *iv;
for chunk in padded.chunks(16) {
let mut buf = [0u8; 16];
for ((b, plain), iv_byte) in buf.iter_mut().zip(chunk.iter()).zip(prev.iter()) {
*b = plain ^ iv_byte;
}
let mut block = GenericArray::clone_from_slice(&buf);
cipher.encrypt_block(&mut block);
output.extend_from_slice(block.as_slice());
prev.copy_from_slice(block.as_slice());
}
output
}
fn aes_256_cbc_encrypt_no_pad(key: &[u8], iv: &[u8; 16], data: &[u8]) -> Vec<u8> {
assert_eq!(key.len(), 32, "AES-256 key must be 32 bytes");
assert!(data.len() % 16 == 0, "plaintext must be block-aligned");
let cipher = Aes256::new_from_slice(key).expect("key length validated");
let mut output = Vec::with_capacity(data.len());
let mut prev: [u8; 16] = *iv;
for chunk in data.chunks(16) {
let mut buf = [0u8; 16];
for ((b, plain), iv_byte) in buf.iter_mut().zip(chunk.iter()).zip(prev.iter()) {
*b = plain ^ iv_byte;
}
let mut block = GenericArray::clone_from_slice(&buf);
cipher.encrypt_block(&mut block);
output.extend_from_slice(block.as_slice());
prev.copy_from_slice(block.as_slice());
}
output
}
pub fn aes_128_cbc_encrypt(key: &[u8], iv: &[u8; 16], plaintext: &[u8]) -> Vec<u8> {
use aes::cipher::BlockEncrypt;
assert_eq!(key.len(), 16, "AES-128 key must be 16 bytes");
let cipher = Aes128::new_from_slice(key).expect("key length validated");
let pad_len = 16 - (plaintext.len() % 16);
let mut padded = Vec::with_capacity(plaintext.len() + pad_len);
padded.extend_from_slice(plaintext);
padded.extend(std::iter::repeat_n(pad_len as u8, pad_len));
let mut output = Vec::with_capacity(16 + padded.len());
output.extend_from_slice(iv);
let mut prev: [u8; 16] = *iv;
for chunk in padded.chunks(16) {
let mut block = [0u8; 16];
for ((b, plain), iv_byte) in block.iter_mut().zip(chunk.iter()).zip(prev.iter()) {
*b = plain ^ iv_byte;
}
let mut arr = GenericArray::clone_from_slice(&block);
cipher.encrypt_block(&mut arr);
output.extend_from_slice(arr.as_slice());
prev.copy_from_slice(arr.as_slice());
}
output
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rc4_empty_input_returns_empty() {
assert_eq!(rc4(b"Key", b""), Vec::<u8>::new());
}
#[test]
fn rc4_matches_known_vector() {
let key = b"Key";
let plaintext = b"Plaintext";
let encrypted = rc4(key, plaintext);
let decrypted = rc4(key, &encrypted);
assert_eq!(decrypted, plaintext);
assert_eq!(
encrypted,
[0xBB, 0xF3, 0x16, 0xE8, 0xD9, 0x40, 0xAF, 0x0A, 0xD3]
);
}
#[test]
fn pad_password_short_pads_with_padding_string() {
let padded = pad_password(b"ab");
assert_eq!(padded[0], b'a');
assert_eq!(padded[1], b'b');
assert_eq!(padded[2], PASSWORD_PADDING[0]);
assert_eq!(padded[31], PASSWORD_PADDING[29]);
}
#[test]
fn pad_password_truncates_to_32_bytes() {
let long = vec![b'x'; 64];
let padded = pad_password(&long);
assert_eq!(padded, [b'x'; 32]);
}
fn build_encrypt_dict_r3(
o_entry: Vec<u8>,
u_entry: Vec<u8>,
permissions: i32,
) -> PdfDictionary {
let mut dict = PdfDictionary::default();
dict.insert("Filter".to_string(), PdfValue::Name("Standard".to_string()));
dict.insert("V".to_string(), PdfValue::Integer(2));
dict.insert("R".to_string(), PdfValue::Integer(3));
dict.insert("Length".to_string(), PdfValue::Integer(128));
dict.insert(
"O".to_string(),
PdfValue::String(crate::types::PdfString(o_entry)),
);
dict.insert(
"U".to_string(),
PdfValue::String(crate::types::PdfString(u_entry)),
);
dict.insert("P".to_string(), PdfValue::Integer(permissions as i64));
dict
}
fn build_r3_handler_inputs(
user_password: &[u8],
owner_password: &[u8],
id_first: &[u8],
) -> (PdfDictionary, Vec<u8>) {
let key_length_bytes = 16;
let permissions: i32 = -4;
let o = test_helpers::compute_o(
owner_password,
user_password,
SecurityRevision::R3,
key_length_bytes,
);
let file_key = test_helpers::compute_file_key(
user_password,
&o,
permissions,
id_first,
key_length_bytes,
);
let u = test_helpers::compute_u_r3(&file_key, id_first);
(build_encrypt_dict_r3(o, u, permissions), file_key)
}
#[test]
fn open_authenticates_user_password() {
let id_first = b"synthetic-id-0123456789abcdef";
let (dict, expected_file_key) = build_r3_handler_inputs(b"userpw", b"ownerpw", id_first);
let handler = StandardSecurityHandler::open(&dict, id_first, b"userpw")
.expect("open succeeds")
.expect("user password authenticates");
assert_eq!(handler.file_key, expected_file_key);
}
#[test]
fn open_authenticates_owner_password() {
let id_first = b"synthetic-id-0123456789abcdef";
let (dict, expected_file_key) = build_r3_handler_inputs(b"userpw", b"ownerpw", id_first);
let handler = StandardSecurityHandler::open(&dict, id_first, b"ownerpw")
.expect("open succeeds")
.expect("owner password authenticates");
assert_eq!(handler.file_key, expected_file_key);
}
#[test]
fn open_rejects_wrong_password() {
let id_first = b"synthetic-id-0123456789abcdef";
let (dict, _) = build_r3_handler_inputs(b"userpw", b"ownerpw", id_first);
let result = StandardSecurityHandler::open(&dict, id_first, b"wrongpw")
.expect("open does not fail, only reports authentication");
assert!(result.is_none());
}
#[test]
fn open_accepts_utf8_password() {
let id_first = b"synthetic-id-0123456789abcdef";
let password = "pässwörd".as_bytes();
let (dict, _) = build_r3_handler_inputs(password, b"ownerpw", id_first);
let handler = StandardSecurityHandler::open(&dict, id_first, password)
.expect("open succeeds")
.expect("UTF-8 password authenticates");
assert_eq!(handler.file_key.len(), 16);
}
fn build_encrypt_dict_v4_aesv2(
o_entry: Vec<u8>,
u_entry: Vec<u8>,
permissions: i32,
encrypt_metadata: Option<bool>,
) -> PdfDictionary {
let mut std_cf = PdfDictionary::default();
std_cf.insert("CFM".to_string(), PdfValue::Name("AESV2".to_string()));
std_cf.insert("Length".to_string(), PdfValue::Integer(16));
std_cf.insert(
"AuthEvent".to_string(),
PdfValue::Name("DocOpen".to_string()),
);
let mut cf = PdfDictionary::default();
cf.insert("StdCF".to_string(), PdfValue::Dictionary(std_cf));
let mut dict = PdfDictionary::default();
dict.insert("Filter".to_string(), PdfValue::Name("Standard".to_string()));
dict.insert("V".to_string(), PdfValue::Integer(4));
dict.insert("R".to_string(), PdfValue::Integer(4));
dict.insert("Length".to_string(), PdfValue::Integer(128));
dict.insert("CF".to_string(), PdfValue::Dictionary(cf));
dict.insert("StmF".to_string(), PdfValue::Name("StdCF".to_string()));
dict.insert("StrF".to_string(), PdfValue::Name("StdCF".to_string()));
dict.insert(
"O".to_string(),
PdfValue::String(crate::types::PdfString(o_entry)),
);
dict.insert(
"U".to_string(),
PdfValue::String(crate::types::PdfString(u_entry)),
);
dict.insert("P".to_string(), PdfValue::Integer(permissions as i64));
if let Some(value) = encrypt_metadata {
dict.insert("EncryptMetadata".to_string(), PdfValue::Bool(value));
}
dict
}
fn build_v4_handler_inputs(
user_password: &[u8],
owner_password: &[u8],
id_first: &[u8],
encrypt_metadata: Option<bool>,
) -> (PdfDictionary, Vec<u8>) {
let permissions: i32 = -4;
let o = test_helpers::compute_o(owner_password, user_password, SecurityRevision::R4, 16);
let file_key = test_helpers::compute_file_key_r4(
user_password,
&o,
permissions,
id_first,
encrypt_metadata.unwrap_or(true),
);
let u = test_helpers::compute_u_r3(&file_key, id_first);
(
build_encrypt_dict_v4_aesv2(o, u, permissions, encrypt_metadata),
file_key,
)
}
#[test]
fn open_v4_aesv2_handler_authenticates_user_password() {
let id_first = b"v4-synthetic-id-0123456789";
let (dict, expected_file_key) =
build_v4_handler_inputs(b"userpw", b"ownerpw", id_first, None);
let handler = StandardSecurityHandler::open(&dict, id_first, b"userpw")
.expect("open succeeds")
.expect("user password authenticates on V=4");
assert_eq!(handler.file_key, expected_file_key);
assert_eq!(handler.string_method, CryptMethod::AesV2);
assert_eq!(handler.stream_method, CryptMethod::AesV2);
assert!(handler.encrypt_metadata);
}
#[test]
fn open_v4_aesv2_handler_authenticates_owner_password() {
let id_first = b"v4-synthetic-id-0123456789";
let (dict, expected_file_key) =
build_v4_handler_inputs(b"userpw", b"ownerpw", id_first, None);
let handler = StandardSecurityHandler::open(&dict, id_first, b"ownerpw")
.expect("open succeeds")
.expect("owner password authenticates on V=4");
assert_eq!(handler.file_key, expected_file_key);
}
#[test]
fn open_v4_honours_encrypt_metadata_false() {
let id_first = b"v4-metadata-id";
let (dict, _) = build_v4_handler_inputs(b"", b"ownerpw", id_first, Some(false));
let handler = StandardSecurityHandler::open(&dict, id_first, b"")
.expect("open succeeds")
.expect("empty password authenticates");
assert!(!handler.encrypts_metadata());
}
#[test]
fn open_v4_identity_crypt_filter_is_passthrough() {
let id_first = b"v4-identity-id";
let (dict_v4, _) = build_v4_handler_inputs(b"", b"ownerpw", id_first, None);
let mut dict = dict_v4;
dict.insert("StrF".to_string(), PdfValue::Name("Identity".to_string()));
dict.insert("StmF".to_string(), PdfValue::Name("Identity".to_string()));
let handler = StandardSecurityHandler::open(&dict, id_first, b"")
.expect("open succeeds")
.expect("empty password authenticates");
assert_eq!(handler.string_method, CryptMethod::Identity);
assert_eq!(handler.stream_method, CryptMethod::Identity);
let ciphertext = b"hello";
let plaintext = handler
.decrypt_bytes(ciphertext, ObjectRef::new(4, 0), BytesKind::Stream)
.expect("identity passes bytes through");
assert_eq!(plaintext, ciphertext);
}
#[test]
fn open_v4_rejects_unsupported_cfm() {
let id_first = b"v4-unsupported-id";
let (dict_v4, _) = build_v4_handler_inputs(b"", b"ownerpw", id_first, None);
let mut dict = dict_v4;
let mut std_cf = PdfDictionary::default();
std_cf.insert("CFM".to_string(), PdfValue::Name("AESV4".to_string()));
std_cf.insert("Length".to_string(), PdfValue::Integer(32));
let mut cf = PdfDictionary::default();
cf.insert("StdCF".to_string(), PdfValue::Dictionary(std_cf));
dict.insert("CF".to_string(), PdfValue::Dictionary(cf));
let error = StandardSecurityHandler::open(&dict, id_first, b"")
.expect_err("unknown CFM must be rejected as unsupported");
assert!(matches!(error, PdfError::Unsupported(_)), "got {error:?}");
}
#[test]
fn aes_128_cbc_round_trip() {
let key = [0x11u8; 16];
let iv = [0x22u8; 16];
let plaintext = b"redact me, please";
let ciphertext = test_helpers::aes_128_cbc_encrypt(&key, &iv, plaintext);
let decrypted = aes_128_cbc_decrypt(&key, &ciphertext).expect("round trip succeeds");
assert_eq!(decrypted, plaintext);
}
#[test]
fn aes_128_cbc_rejects_bad_pkcs7_padding() {
let key = [0x11u8; 16];
let iv = [0x22u8; 16];
let plaintext = b"abcdef";
let mut ciphertext = test_helpers::aes_128_cbc_encrypt(&key, &iv, plaintext);
let last = ciphertext.len() - 1;
ciphertext[last] ^= 0xFF;
let error =
aes_128_cbc_decrypt(&key, &ciphertext).expect_err("corrupted padding must be rejected");
assert!(matches!(error, PdfError::Corrupt(_)), "got {error:?}");
}
#[test]
fn aes_128_cbc_rejects_short_ciphertext() {
let key = [0x11u8; 16];
let error = aes_128_cbc_decrypt(&key, &[0u8; 16])
.expect_err("ciphertext shorter than IV+1 block must be rejected");
assert!(matches!(error, PdfError::Corrupt(_)), "got {error:?}");
}
fn build_encrypt_dict_v5_aesv3(
o: Vec<u8>,
u: Vec<u8>,
oe: Vec<u8>,
ue: Vec<u8>,
permissions: i32,
perms: Option<Vec<u8>>,
revision: SecurityRevision,
) -> PdfDictionary {
let mut std_cf = PdfDictionary::default();
std_cf.insert("CFM".to_string(), PdfValue::Name("AESV3".to_string()));
std_cf.insert("Length".to_string(), PdfValue::Integer(32));
std_cf.insert(
"AuthEvent".to_string(),
PdfValue::Name("DocOpen".to_string()),
);
let mut cf = PdfDictionary::default();
cf.insert("StdCF".to_string(), PdfValue::Dictionary(std_cf));
let r_value = match revision {
SecurityRevision::R5 => 5,
SecurityRevision::R6 => 6,
_ => panic!("test helper only supports R5 / R6"),
};
let mut dict = PdfDictionary::default();
dict.insert("Filter".to_string(), PdfValue::Name("Standard".to_string()));
dict.insert("V".to_string(), PdfValue::Integer(5));
dict.insert("R".to_string(), PdfValue::Integer(r_value));
dict.insert("Length".to_string(), PdfValue::Integer(256));
dict.insert("CF".to_string(), PdfValue::Dictionary(cf));
dict.insert("StmF".to_string(), PdfValue::Name("StdCF".to_string()));
dict.insert("StrF".to_string(), PdfValue::Name("StdCF".to_string()));
dict.insert(
"O".to_string(),
PdfValue::String(crate::types::PdfString(o)),
);
dict.insert(
"U".to_string(),
PdfValue::String(crate::types::PdfString(u)),
);
dict.insert(
"OE".to_string(),
PdfValue::String(crate::types::PdfString(oe)),
);
dict.insert(
"UE".to_string(),
PdfValue::String(crate::types::PdfString(ue)),
);
dict.insert("P".to_string(), PdfValue::Integer(permissions as i64));
if let Some(value) = perms {
dict.insert(
"Perms".to_string(),
PdfValue::String(crate::types::PdfString(value)),
);
}
dict
}
fn build_v5_handler_inputs(
user_password: &[u8],
owner_password: &[u8],
revision: SecurityRevision,
) -> (PdfDictionary, [u8; 32]) {
let file_key = [0x13u8; 32];
let u_validation_salt = [0xAAu8; 8];
let u_key_salt = [0xBBu8; 8];
let o_validation_salt = [0xCCu8; 8];
let o_key_salt = [0xDDu8; 8];
let (u, ue) = test_helpers::compute_v5_u_and_ue(
user_password,
&u_validation_salt,
&u_key_salt,
&file_key,
revision,
);
let u_vector: [u8; 48] = u.as_slice().try_into().expect("U is 48 bytes");
let (o, oe) = test_helpers::compute_v5_o_and_oe(
owner_password,
&o_validation_salt,
&o_key_salt,
&u_vector,
&file_key,
revision,
);
(
build_encrypt_dict_v5_aesv3(o, u, oe, ue, -4, None, revision),
file_key,
)
}
#[test]
fn open_v5_r6_authenticates_user_password() {
let (dict, expected_file_key) =
build_v5_handler_inputs(b"userpw", b"ownerpw", SecurityRevision::R6);
let handler = StandardSecurityHandler::open(&dict, b"", b"userpw")
.expect("open succeeds")
.expect("user password authenticates on V=5 / R=6");
assert_eq!(handler.file_key, expected_file_key);
assert_eq!(handler.string_method, CryptMethod::AesV3);
assert_eq!(handler.stream_method, CryptMethod::AesV3);
}
#[test]
fn open_v5_r6_authenticates_owner_password() {
let (dict, expected_file_key) =
build_v5_handler_inputs(b"userpw", b"ownerpw", SecurityRevision::R6);
let handler = StandardSecurityHandler::open(&dict, b"", b"ownerpw")
.expect("open succeeds")
.expect("owner password authenticates on V=5 / R=6");
assert_eq!(handler.file_key, expected_file_key);
}
#[test]
fn open_v5_r6_rejects_wrong_password() {
let (dict, _) = build_v5_handler_inputs(b"userpw", b"ownerpw", SecurityRevision::R6);
let result = StandardSecurityHandler::open(&dict, b"", b"wrongpw")
.expect("open does not fail, only reports authentication");
assert!(result.is_none());
}
#[test]
fn open_v5_r5_authenticates_user_password() {
let (dict, expected_file_key) =
build_v5_handler_inputs(b"userpw", b"ownerpw", SecurityRevision::R5);
let handler = StandardSecurityHandler::open(&dict, b"", b"userpw")
.expect("open succeeds")
.expect("user password authenticates on V=5 / R=5");
assert_eq!(handler.file_key, expected_file_key);
}
#[test]
fn open_v5_r5_empty_password_authenticates() {
let (dict, _) = build_v5_handler_inputs(b"", b"ownerpw", SecurityRevision::R5);
let handler = StandardSecurityHandler::open(&dict, b"", b"")
.expect("open succeeds")
.expect("empty password authenticates on V=5 / R=5");
assert_eq!(handler.string_method, CryptMethod::AesV3);
}
#[test]
fn aes_256_cbc_round_trip_through_handler() {
let key = [0x13u8; 32];
let iv = [0x77u8; 16];
let plaintext = b"top secret V=5 content";
let ciphertext = test_helpers::aes_256_cbc_encrypt(&key, &iv, plaintext);
let decrypted = aes_256_cbc_decrypt(&key, &ciphertext).expect("round trip succeeds");
assert_eq!(decrypted, plaintext);
}
#[test]
fn open_r2_authenticates_owner_password() {
let id_first = b"r2-synthetic-id";
let user_password = b"u2";
let owner_password = b"o2";
let key_length_bytes = 5; let permissions: i32 = -4;
let o = test_helpers::compute_o(
owner_password,
user_password,
SecurityRevision::R2,
key_length_bytes,
);
let file_key = test_helpers::compute_file_key_with_revision(
user_password,
&o,
permissions,
id_first,
key_length_bytes,
SecurityRevision::R2,
);
let u = test_helpers::rc4(&file_key, &PASSWORD_PADDING);
let mut dict = PdfDictionary::default();
dict.insert("Filter".to_string(), PdfValue::Name("Standard".to_string()));
dict.insert("V".to_string(), PdfValue::Integer(1));
dict.insert("R".to_string(), PdfValue::Integer(2));
dict.insert("Length".to_string(), PdfValue::Integer(40));
dict.insert(
"O".to_string(),
PdfValue::String(crate::types::PdfString(o)),
);
dict.insert(
"U".to_string(),
PdfValue::String(crate::types::PdfString(u)),
);
dict.insert("P".to_string(), PdfValue::Integer(permissions as i64));
let handler = StandardSecurityHandler::open(&dict, id_first, owner_password)
.expect("open succeeds")
.expect("owner password authenticates on R=2");
assert_eq!(handler.file_key, file_key);
}
}