Skip to main content

rustium_pdf/
crypt.rs

1//! The standard security handler: RC4 and AES, revisions 2 through 6.
2//!
3//! Everything here serves `Decryptor`, which the file parser consults while loading objects.
4//! Only password-based encryption is implemented; certificate-based files are refused.
5
6use aes::cipher::{block_padding::NoPadding, BlockDecryptMut, BlockEncryptMut, KeyIvInit};
7use md5::{Digest, Md5};
8use sha2::{Sha256, Sha384, Sha512};
9
10use crate::error::{Error, Result};
11use crate::object::{Dict, Object};
12
13const PAD: [u8; 32] = [
14    0x28, 0xBF, 0x4E, 0x5E, 0x4E, 0x75, 0x8A, 0x41, 0x64, 0x00, 0x4E, 0x56, 0xFF, 0xFA, 0x01, 0x08,
15    0x2E, 0x2E, 0x00, 0xB6, 0xD0, 0x68, 0x3E, 0x80, 0x2F, 0x0C, 0xA9, 0xFE, 0x64, 0x53, 0x69, 0x7A,
16];
17
18#[derive(Clone, Copy, PartialEq)]
19enum Cipher {
20    Rc4,
21    Aes,
22    Identity,
23}
24
25pub struct Decryptor {
26    key: Vec<u8>,
27    string_cipher: Cipher,
28    stream_cipher: Cipher,
29    /// V5 uses the file key directly; earlier versions derive a per-object key.
30    per_object_key: bool,
31    encrypt_metadata: bool,
32}
33
34impl Decryptor {
35    pub fn new(
36        enc: &Dict,
37        file_id: &[u8],
38        password: Option<&str>,
39        resolve: &dyn Fn(&Object) -> Object,
40    ) -> Result<Self> {
41        let get = |k: &str| enc.get(k).map(resolve);
42        let filter = get("Filter").and_then(|o| o.as_name().map(str::to_owned));
43        if filter.as_deref() != Some("Standard") {
44            return Err(Error::UnsupportedEncryption(format!(
45                "security handler {:?}",
46                filter.unwrap_or_default()
47            )));
48        }
49        let v = get("V").and_then(|o| o.as_int()).unwrap_or(0);
50        let r = get("R").and_then(|o| o.as_int()).unwrap_or(0);
51        let length_bits = get("Length").and_then(|o| o.as_int()).unwrap_or(40);
52        let o_entry = get("O")
53            .and_then(|o| o.as_string().map(<[u8]>::to_vec))
54            .unwrap_or_default();
55        let u_entry = get("U")
56            .and_then(|o| o.as_string().map(<[u8]>::to_vec))
57            .unwrap_or_default();
58        let p = get("P").and_then(|o| o.as_int()).unwrap_or(-1) as i32;
59        let encrypt_metadata = get("EncryptMetadata")
60            .and_then(|o| o.as_bool())
61            .unwrap_or(true);
62        let password = password.unwrap_or("");
63
64        // Crypt-filter ciphers for V4/V5; earlier versions are RC4 throughout.
65        let (string_cipher, stream_cipher) = if v >= 4 {
66            let cf = get("CF")
67                .and_then(|o| o.as_dict().cloned())
68                .unwrap_or_default();
69            let cipher_of = |name: Option<String>| -> Cipher {
70                let Some(name) = name else {
71                    return Cipher::Identity;
72                };
73                if name == "Identity" {
74                    return Cipher::Identity;
75                }
76                let cfm = cf
77                    .get(&name)
78                    .map(resolve)
79                    .and_then(|o| o.as_dict().and_then(|d| d.get("CFM").map(resolve)))
80                    .and_then(|o| o.as_name().map(str::to_owned));
81                match cfm.as_deref() {
82                    Some("V2") => Cipher::Rc4,
83                    Some("AESV2") | Some("AESV3") => Cipher::Aes,
84                    _ => Cipher::Identity,
85                }
86            };
87            let strf = get("StrF").and_then(|o| o.as_name().map(str::to_owned));
88            let stmf = get("StmF").and_then(|o| o.as_name().map(str::to_owned));
89            (cipher_of(strf), cipher_of(stmf))
90        } else {
91            (Cipher::Rc4, Cipher::Rc4)
92        };
93
94        if v == 5 || r >= 5 {
95            let oe = get("OE")
96                .and_then(|o| o.as_string().map(<[u8]>::to_vec))
97                .unwrap_or_default();
98            let ue = get("UE")
99                .and_then(|o| o.as_string().map(<[u8]>::to_vec))
100                .unwrap_or_default();
101            let key = derive_key_v5(password, r, &o_entry, &u_entry, &oe, &ue)?;
102            return Ok(Self {
103                key,
104                string_cipher,
105                stream_cipher,
106                per_object_key: false,
107                encrypt_metadata,
108            });
109        }
110
111        if !(1..=4).contains(&v) {
112            return Err(Error::UnsupportedEncryption(format!("V {v}")));
113        }
114        let key_len = if r == 2 {
115            5
116        } else {
117            (length_bits / 8).clamp(5, 16) as usize
118        };
119
120        // Try the password as the user password, then as the owner password.
121        let user_key = legacy_file_key(
122            password.as_bytes(),
123            &o_entry,
124            p,
125            file_id,
126            r,
127            key_len,
128            encrypt_metadata,
129        );
130        if legacy_check_user(&user_key, &u_entry, file_id, r) {
131            return Ok(Self {
132                key: user_key,
133                string_cipher,
134                stream_cipher,
135                per_object_key: true,
136                encrypt_metadata,
137            });
138        }
139        let recovered = legacy_owner_to_user(password.as_bytes(), &o_entry, r, key_len);
140        let owner_key = legacy_file_key(
141            &recovered,
142            &o_entry,
143            p,
144            file_id,
145            r,
146            key_len,
147            encrypt_metadata,
148        );
149        if legacy_check_user(&owner_key, &u_entry, file_id, r) {
150            return Ok(Self {
151                key: owner_key,
152                string_cipher,
153                stream_cipher,
154                per_object_key: true,
155                encrypt_metadata,
156            });
157        }
158        Err(Error::Password)
159    }
160
161    fn object_key(&self, num: u32, gen: u16, aes: bool) -> Vec<u8> {
162        if !self.per_object_key {
163            return self.key.clone();
164        }
165        let mut h = Md5::new();
166        h.update(&self.key);
167        h.update(&num.to_le_bytes()[..3]);
168        h.update(&gen.to_le_bytes()[..2]);
169        if aes {
170            h.update(b"sAlT");
171        }
172        let digest = h.finalize();
173        let n = (self.key.len() + 5).min(16);
174        digest[..n].to_vec()
175    }
176
177    pub fn decrypt_string(&self, num: u32, gen: u16, data: &[u8]) -> Vec<u8> {
178        self.decrypt(self.string_cipher, num, gen, data)
179    }
180
181    pub fn decrypt_stream(&self, num: u32, gen: u16, data: &[u8]) -> Vec<u8> {
182        self.decrypt(self.stream_cipher, num, gen, data)
183    }
184
185    /// Whether a stream of the given `/Type` is encrypted at all.
186    pub fn stream_needs_decrypt(&self, stream_type: Option<&str>) -> bool {
187        // A document may declare that its metadata stays in the clear; every other stream is
188        // encrypted. Naming the exemption keeps the negation readable.
189        let metadata_exempt = stream_type == Some("Metadata") && !self.encrypt_metadata;
190        !metadata_exempt
191    }
192
193    fn decrypt(&self, cipher: Cipher, num: u32, gen: u16, data: &[u8]) -> Vec<u8> {
194        match cipher {
195            Cipher::Identity => data.to_vec(),
196            Cipher::Rc4 => rc4(&self.object_key(num, gen, false), data),
197            Cipher::Aes => aes_cbc_decrypt(&self.object_key(num, gen, true), data),
198        }
199    }
200}
201
202// ---- legacy (R2-R4) key derivation ---------------------------------------------------------
203
204fn padded(pw: &[u8]) -> [u8; 32] {
205    let mut out = [0u8; 32];
206    let n = pw.len().min(32);
207    out[..n].copy_from_slice(&pw[..n]);
208    out[n..].copy_from_slice(&PAD[..32 - n]);
209    out
210}
211
212fn legacy_file_key(
213    password: &[u8],
214    o_entry: &[u8],
215    p: i32,
216    file_id: &[u8],
217    r: i64,
218    key_len: usize,
219    encrypt_metadata: bool,
220) -> Vec<u8> {
221    let mut h = Md5::new();
222    h.update(padded(password));
223    h.update(&o_entry[..o_entry.len().min(32)]);
224    h.update(p.to_le_bytes());
225    h.update(file_id);
226    if r >= 4 && !encrypt_metadata {
227        h.update([0xFF, 0xFF, 0xFF, 0xFF]);
228    }
229    let mut key = h.finalize()[..].to_vec();
230    if r >= 3 {
231        for _ in 0..50 {
232            key = Md5::digest(&key[..key_len])[..].to_vec();
233        }
234    }
235    key.truncate(key_len);
236    key
237}
238
239fn legacy_check_user(key: &[u8], u_entry: &[u8], file_id: &[u8], r: i64) -> bool {
240    if u_entry.len() < 16 {
241        return false;
242    }
243    if r == 2 {
244        return rc4(key, &PAD) == u_entry[..32.min(u_entry.len())];
245    }
246    let mut h = Md5::new();
247    h.update(PAD);
248    h.update(file_id);
249    let mut val = rc4(key, &h.finalize());
250    for i in 1..=19u8 {
251        let k: Vec<u8> = key.iter().map(|&b| b ^ i).collect();
252        val = rc4(&k, &val);
253    }
254    val[..16] == u_entry[..16]
255}
256
257/// Algorithm 7 in reverse: recovers the (padded) user password from `O` given the owner
258/// password. Returns garbage for a wrong password, which the user check then rejects.
259fn legacy_owner_to_user(owner_pw: &[u8], o_entry: &[u8], r: i64, key_len: usize) -> Vec<u8> {
260    let mut key = Md5::digest(padded(owner_pw))[..].to_vec();
261    if r >= 3 {
262        for _ in 0..50 {
263            key = Md5::digest(&key)[..].to_vec();
264        }
265    }
266    key.truncate(key_len);
267    let mut val = o_entry[..o_entry.len().min(32)].to_vec();
268    if r == 2 {
269        val = rc4(&key, &val);
270    } else {
271        for i in (0..=19u8).rev() {
272            let k: Vec<u8> = key.iter().map(|&b| b ^ i).collect();
273            val = rc4(&k, &val);
274        }
275    }
276    val
277}
278
279// ---- V5 / AES-256 (R5, R6) -----------------------------------------------------------------
280
281fn derive_key_v5(
282    password: &str,
283    r: i64,
284    o_entry: &[u8],
285    u_entry: &[u8],
286    oe: &[u8],
287    ue: &[u8],
288) -> Result<Vec<u8>> {
289    // SASLprep is required by spec for passwords; ASCII passwords (the overwhelming case) are
290    // unaffected. Non-ASCII passwords are passed through as UTF-8.
291    let pw = {
292        let b = password.as_bytes();
293        &b[..b.len().min(127)]
294    };
295    if o_entry.len() < 48 || u_entry.len() < 48 {
296        return Err(Error::UnsupportedEncryption("short O/U entries".into()));
297    }
298
299    let hash = |pw: &[u8], salt: &[u8], udata: &[u8]| -> [u8; 32] {
300        if r == 5 {
301            let mut h = Sha256::new();
302            h.update(pw);
303            h.update(salt);
304            h.update(udata);
305            h.finalize().into()
306        } else {
307            hash_2b(pw, salt, udata)
308        }
309    };
310
311    // Try owner password first: its hash covers the whole U entry.
312    let (o_hash, o_vsalt, o_ksalt) = (&o_entry[..32], &o_entry[32..40], &o_entry[40..48]);
313    let (u_hash, u_vsalt, u_ksalt) = (&u_entry[..32], &u_entry[32..40], &u_entry[40..48]);
314
315    if hash(pw, o_vsalt, &u_entry[..48]) == *o_hash {
316        let ikey = hash(pw, o_ksalt, &u_entry[..48]);
317        let key = aes256_cbc_no_pad_decrypt(&ikey, &[0u8; 16], oe)?;
318        return Ok(key);
319    }
320    if hash(pw, u_vsalt, &[]) == *u_hash {
321        let ikey = hash(pw, u_ksalt, &[]);
322        let key = aes256_cbc_no_pad_decrypt(&ikey, &[0u8; 16], ue)?;
323        return Ok(key);
324    }
325    Err(Error::Password)
326}
327
328/// ISO 32000-2 algorithm 2.B: the hardened iterated hash used by revision 6.
329fn hash_2b(pw: &[u8], salt: &[u8], udata: &[u8]) -> [u8; 32] {
330    let mut k: Vec<u8> = {
331        let mut h = Sha256::new();
332        h.update(pw);
333        h.update(salt);
334        h.update(udata);
335        h.finalize().to_vec()
336    };
337    let mut round = 0usize;
338    loop {
339        // K1 = (pw || K || udata) repeated 64 times.
340        let mut unit = Vec::with_capacity(pw.len() + k.len() + udata.len());
341        unit.extend_from_slice(pw);
342        unit.extend_from_slice(&k);
343        unit.extend_from_slice(udata);
344        let mut k1 = Vec::with_capacity(unit.len() * 64);
345        for _ in 0..64 {
346            k1.extend_from_slice(&unit);
347        }
348        // E = AES-128-CBC(key=K[0..16], iv=K[16..32], K1)
349        let enc = Aes128CbcEnc::new_from_slices(&k[..16], &k[16..32]).expect("fixed sizes");
350        let e = enc.encrypt_padded_vec_mut::<NoPadding>(&k1);
351        let modulus = e[..16].iter().map(|&b| b as u32).sum::<u32>() % 3;
352        k = match modulus {
353            0 => Sha256::digest(&e).to_vec(),
354            1 => Sha384::digest(&e).to_vec(),
355            _ => Sha512::digest(&e).to_vec(),
356        };
357        round += 1;
358        if round >= 64 && (*e.last().unwrap() as usize) <= round - 32 {
359            break;
360        }
361    }
362    let mut out = [0u8; 32];
363    out.copy_from_slice(&k[..32]);
364    out
365}
366
367// ---- primitives ----------------------------------------------------------------------------
368
369type Aes128CbcEnc = cbc::Encryptor<aes::Aes128>;
370type Aes128CbcDec = cbc::Decryptor<aes::Aes128>;
371type Aes256CbcDec = cbc::Decryptor<aes::Aes256>;
372
373pub fn rc4(key: &[u8], data: &[u8]) -> Vec<u8> {
374    if key.is_empty() {
375        // RC4 has no empty-key schedule; a malformed /Encrypt dict must not panic the reader.
376        return data.to_vec();
377    }
378    let mut s: [u8; 256] = std::array::from_fn(|i| i as u8);
379    let mut j = 0u8;
380    for i in 0..256 {
381        j = j.wrapping_add(s[i]).wrapping_add(key[i % key.len()]);
382        s.swap(i, j as usize);
383    }
384    let (mut i, mut j) = (0u8, 0u8);
385    data.iter()
386        .map(|&b| {
387            i = i.wrapping_add(1);
388            j = j.wrapping_add(s[i as usize]);
389            s.swap(i as usize, j as usize);
390            b ^ s[(s[i as usize].wrapping_add(s[j as usize])) as usize]
391        })
392        .collect()
393}
394
395/// AES-CBC with the IV in the first block, PKCS#5 padding stripped leniently.
396fn aes_cbc_decrypt(key: &[u8], data: &[u8]) -> Vec<u8> {
397    if data.len() < 32 || !(data.len() - 16).is_multiple_of(16) {
398        return Vec::new();
399    }
400    let (iv, body) = data.split_at(16);
401    let mut buf = body.to_vec();
402    let ok = match key.len() {
403        16 => Aes128CbcDec::new_from_slices(key, iv)
404            .map(|c| c.decrypt_padded_mut::<NoPadding>(&mut buf).map(|_| ()))
405            .is_ok_and(|r| r.is_ok()),
406        32 => Aes256CbcDec::new_from_slices(key, iv)
407            .map(|c| c.decrypt_padded_mut::<NoPadding>(&mut buf).map(|_| ()))
408            .is_ok_and(|r| r.is_ok()),
409        _ => false,
410    };
411    if !ok {
412        return Vec::new();
413    }
414    // Strip padding when it is well-formed; tolerate producers that omit it.
415    if let Some(&pad) = buf.last() {
416        if (1..=16).contains(&pad) && buf.len() >= pad as usize {
417            buf.truncate(buf.len() - pad as usize);
418        }
419    }
420    buf
421}
422
423fn aes256_cbc_no_pad_decrypt(key: &[u8; 32], iv: &[u8; 16], data: &[u8]) -> Result<Vec<u8>> {
424    if !data.len().is_multiple_of(16) || data.is_empty() {
425        return Err(Error::UnsupportedEncryption("bad OE/UE length".into()));
426    }
427    let mut buf = data.to_vec();
428    Aes256CbcDec::new_from_slices(key, iv)
429        .map_err(|_| Error::UnsupportedEncryption("aes-256 init".into()))?
430        .decrypt_padded_mut::<NoPadding>(&mut buf)
431        .map_err(|_| Error::UnsupportedEncryption("aes-256 decrypt".into()))?;
432    Ok(buf)
433}
434
435#[cfg(test)]
436mod tests {
437    use super::*;
438
439    #[test]
440    fn rc4_test_vectors() {
441        // RFC 6229-style known vector: key "Key", plaintext "Plaintext".
442        let out = rc4(b"Key", b"Plaintext");
443        assert_eq!(out, [0xBB, 0xF3, 0x16, 0xE8, 0xD9, 0x40, 0xAF, 0x0A, 0xD3]);
444        // Round trip.
445        assert_eq!(rc4(b"Key", &out), b"Plaintext");
446    }
447
448    #[test]
449    fn padding_pads_and_truncates() {
450        assert_eq!(padded(b"")[..], PAD[..]);
451        let p = padded(b"secret");
452        assert_eq!(&p[..6], b"secret");
453        assert_eq!(&p[6..], &PAD[..26]);
454    }
455}