ese_parser/
lib.rs

1//! **ese-parser** — Microsoft Extensible Storage Engine (ESE / "Jet Blue") file parser
2//! in pure Rust.
3//!
4//! ESE is the on-disk B-tree database Windows uses for anything that needs local persistence
5//! with recovery: `ntds.dit` (Active Directory), `WebCacheV*.dat` (Edge/IE history),
6//! `Windows.edb` (Search index), `Mail.dat` (legacy Exchange), event-tracing logs.
7//!
8//! This crate is a **from-scratch** parser (no C-lib FFI) focused on read-only forensic /
9//! offensive-security use — the subset a defender or attacker needs to extract data from a
10//! captured `.dit`/`.edb`, not the full writable Jet API.
11//!
12//! # v0.1 scope
13//!
14//! - [`EseFile::open`] — parse the 668-byte database header (magic, format-version,
15//!   page-size, file-type, previous-full-backup / previous-incremental-backup metadata,
16//!   the min/max log-generation numbers required to know if the DB is consistent).
17//! - [`EseFile::page`] — random-access read of a specific page by page number, returning a
18//!   [`RawPage`] with parsed header fields (page number, previous/next page in the B-tree
19//!   sibling chain, page flags telling you whether it's a leaf/branch/root/parent-of-leaf).
20//! - [`RawPage`] constants — [`page_flag`] bits, [`tag`] flag bits.
21//!
22//! # v0.2 roadmap
23//!
24//! - B-tree walk (root → branch → leaf, following child page pointers).
25//! - Catalog table decode — the metadata table that lists every user table + its column
26//!   definitions.
27//! - Row / tag decode with long-value + multi-value reassembly.
28//! - Then: this becomes the backend for
29//!   [`ntds-parse`](https://crates.io/crates/ntds-parse) `EseBackend` trait so callers can
30//!   `for row in ntds.datatable()`.
31//!
32//! # Non-goals
33//!
34//! - Writable Jet API (this is read-only).
35//! - Recovery from a dirty shutdown (defer that to Windows' own `esentutl /p`).
36//! - Encrypted databases (Windows database encryption uses per-page XOR with a table key —
37//!   out of scope for v0.1).
38//!
39//! # Verification
40//!
41//! Header offsets validated against
42//! [libesedb's spec doc](https://github.com/libyal/libesedb/blob/main/documentation/Extensible%20Storage%20Engine%20(ESE)%20Database%20File%20(EDB)%20format.asciidoc).
43//! Page header layout matches Joachim Metz's reverse-engineering notes.
44
45use std::fs::File;
46use std::io::{Read, Seek, SeekFrom};
47use std::path::Path;
48
49/// The 4-byte little-endian file signature every ESE database starts with (offset 4).
50pub const ESE_FILE_SIGNATURE: u32 = 0x89AB_CDEF;
51
52/// File-type values in the database header (offset 12).
53pub mod file_type {
54    pub const DATABASE: u32 = 0;
55    pub const STREAMING_FILE: u32 = 1;
56}
57
58/// Page-header flag bits ([MS-ESE] and libesedb reference).
59pub mod page_flag {
60    /// Page is a root page of a B-tree.
61    pub const ROOT: u32 = 0x0000_0001;
62    /// Page is a leaf page (holds data records).
63    pub const LEAF: u32 = 0x0000_0002;
64    /// Page is a parent-of-leaf (its child pages are all leaves).
65    pub const PARENT: u32 = 0x0000_0004;
66    /// Page is empty (fresh allocation).
67    pub const EMPTY: u32 = 0x0000_0008;
68    /// Page contains a space tree node.
69    pub const SPACE_TREE: u32 = 0x0000_0020;
70    /// Page belongs to an index B-tree (not the primary/data tree).
71    pub const INDEX: u32 = 0x0000_0040;
72    /// Page is part of a long-value chunk chain.
73    pub const LONG_VALUE: u32 = 0x0000_0080;
74    /// Page has been written to since the last full-database backup.
75    pub const NEW_RECORD_FORMAT: u32 = 0x0000_2000;
76    /// Page has scrubbed data (deleted rows overwritten).
77    pub const SCRUBBED: u32 = 0x0000_4000;
78    /// Page has a new checksum format (post-Windows Vista).
79    pub const NEW_CHECKSUM_FORMAT: u32 = 0x0000_8000;
80}
81
82/// A minimally-parsed ESE database file — reads the header + gives you random-access page
83/// reads. Does not walk B-trees or enumerate tables (v0.2 roadmap).
84pub struct EseFile {
85    file: File,
86    format_version: u32,
87    format_revision: u32,
88    page_size: u32,
89    file_type: u32,
90    previous_incremental_backup_log_gen: u32,
91    previous_full_backup_log_gen: u32,
92    current_full_backup_log_gen: u32,
93}
94
95impl EseFile {
96    /// Open + validate an ESE database file. Reads only the 668-byte database header at
97    /// open-time; page reads happen on demand.
98    pub fn open(path: impl AsRef<Path>) -> Result<Self, Error> {
99        let mut file = File::open(path).map_err(Error::Io)?;
100        let mut hdr = [0u8; 668];
101        file.read_exact(&mut hdr).map_err(Error::Io)?;
102        let parsed = Self::parse_header_fields(&hdr)?;
103        Ok(EseFile {
104            file,
105            format_version: parsed.format_version,
106            format_revision: parsed.format_revision,
107            page_size: parsed.page_size,
108            file_type: parsed.file_type,
109            previous_incremental_backup_log_gen: parsed.previous_incremental_backup_log_gen,
110            previous_full_backup_log_gen: parsed.previous_full_backup_log_gen,
111            current_full_backup_log_gen: parsed.current_full_backup_log_gen,
112        })
113    }
114
115    /// Parse the 668-byte header without any I/O. Useful for tests or when the caller has
116    /// already read the bytes over the wire.
117    pub fn parse_header(hdr: &[u8]) -> Result<HeaderFields, Error> {
118        Self::parse_header_fields(hdr)
119    }
120
121    fn parse_header_fields(hdr: &[u8]) -> Result<HeaderFields, Error> {
122        if hdr.len() < 668 {
123            return Err(Error::Format(format!(
124                "header too short: {} bytes (want 668)",
125                hdr.len()
126            )));
127        }
128        let magic = u32::from_le_bytes([hdr[4], hdr[5], hdr[6], hdr[7]]);
129        if magic != ESE_FILE_SIGNATURE {
130            return Err(Error::Format(format!(
131                "bad ESE magic: got {magic:#010x}, want {ESE_FILE_SIGNATURE:#010x}"
132            )));
133        }
134        let format_version = u32::from_le_bytes([hdr[8], hdr[9], hdr[10], hdr[11]]);
135        let file_type = u32::from_le_bytes([hdr[12], hdr[13], hdr[14], hdr[15]]);
136        let page_size = u32::from_le_bytes([hdr[236], hdr[237], hdr[238], hdr[239]]);
137        if ![2048, 4096, 8192, 16384, 32768].contains(&page_size) {
138            return Err(Error::Format(format!("odd page size: {page_size}")));
139        }
140        let format_revision = u32::from_le_bytes([hdr[240], hdr[241], hdr[242], hdr[243]]);
141        // Backup log-gen fields — used to detect a dirty shutdown, the caller can decide
142        // whether to bail on inconsistent files.
143        // Offsets per libesedb spec:
144        //   248..260 = previous full backup snapshot
145        //   260..272 = previous incremental backup snapshot
146        //   272..284 = current full backup snapshot
147        // Each snapshot is a LOG_POSITION struct (8-byte LSN + 4-byte log-gen); the log-gen
148        // is at snapshot base + 8.
149        let previous_full_backup_log_gen =
150            u32::from_le_bytes([hdr[256], hdr[257], hdr[258], hdr[259]]);
151        let previous_incremental_backup_log_gen =
152            u32::from_le_bytes([hdr[268], hdr[269], hdr[270], hdr[271]]);
153        let current_full_backup_log_gen =
154            u32::from_le_bytes([hdr[280], hdr[281], hdr[282], hdr[283]]);
155        Ok(HeaderFields {
156            format_version,
157            format_revision,
158            page_size,
159            file_type,
160            previous_full_backup_log_gen,
161            previous_incremental_backup_log_gen,
162            current_full_backup_log_gen,
163        })
164    }
165
166    pub fn format_version(&self) -> u32 {
167        self.format_version
168    }
169    pub fn format_revision(&self) -> u32 {
170        self.format_revision
171    }
172    pub fn page_size(&self) -> u32 {
173        self.page_size
174    }
175    pub fn file_type(&self) -> u32 {
176        self.file_type
177    }
178    pub fn previous_full_backup_log_gen(&self) -> u32 {
179        self.previous_full_backup_log_gen
180    }
181    pub fn previous_incremental_backup_log_gen(&self) -> u32 {
182        self.previous_incremental_backup_log_gen
183    }
184    pub fn current_full_backup_log_gen(&self) -> u32 {
185        self.current_full_backup_log_gen
186    }
187
188    /// Read a page from the database by 1-indexed page number (page 0 is the header, page 1
189    /// is the shadow header, real pages start at 2). Returns the parsed [`RawPage`].
190    pub fn page(&mut self, page_number: u32) -> Result<RawPage, Error> {
191        let offset = page_number as u64 * self.page_size as u64;
192        self.file.seek(SeekFrom::Start(offset)).map_err(Error::Io)?;
193        let mut buf = vec![0u8; self.page_size as usize];
194        self.file.read_exact(&mut buf).map_err(Error::Io)?;
195        RawPage::parse(page_number, &buf, self.page_size)
196    }
197}
198
199/// Result of parsing the 668-byte header without holding onto the file. Callers use this
200/// when they want to display + validate header metadata without keeping an open handle.
201#[derive(Debug, Clone, Copy)]
202#[non_exhaustive]
203pub struct HeaderFields {
204    pub format_version: u32,
205    pub format_revision: u32,
206    pub page_size: u32,
207    pub file_type: u32,
208    pub previous_full_backup_log_gen: u32,
209    pub previous_incremental_backup_log_gen: u32,
210    pub current_full_backup_log_gen: u32,
211}
212
213/// One page read out of the ESE database. Header parsed; page-body bytes retained so
214/// downstream (a future B-tree walker) can decode tags on demand.
215#[derive(Clone)]
216#[non_exhaustive]
217pub struct RawPage {
218    /// 1-indexed page number.
219    pub page_number: u32,
220    /// Bitmask of [`page_flag`] flags.
221    pub page_flags: u32,
222    /// Page number of the previous sibling in the B-tree chain (0 = none).
223    pub previous_page: u32,
224    /// Page number of the next sibling in the B-tree chain (0 = none).
225    pub next_page: u32,
226    /// Father Data Page (the parent B-tree node's page number).
227    pub father_data_page: u32,
228    /// Object identifier this page belongs to (the FDP object ID).
229    pub object_id: u32,
230    /// Available data size (bytes free for new tags on this page).
231    pub available_data_size: u16,
232    /// Number of tag entries on this page.
233    pub available_page_tag: u16,
234    /// The full page bytes (retained so B-tree/tag decoders can work on top).
235    pub bytes: Vec<u8>,
236}
237
238impl std::fmt::Debug for RawPage {
239    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
240        f.debug_struct("RawPage")
241            .field("page_number", &self.page_number)
242            .field("page_flags", &format_args!("{:#010x}", self.page_flags))
243            .field("previous_page", &self.previous_page)
244            .field("next_page", &self.next_page)
245            .field("father_data_page", &self.father_data_page)
246            .field("object_id", &self.object_id)
247            .field("available_data_size", &self.available_data_size)
248            .field("available_page_tag", &self.available_page_tag)
249            .field("bytes_len", &self.bytes.len())
250            .finish()
251    }
252}
253
254impl RawPage {
255    /// Parse a page's header. The layout differs between the "old" (pre-Vista, page-size-
256    /// specific) and "new" (post-Vista, standard 40-byte header) formats. v0.1 supports the
257    /// new format that Server 2008+ NTDS.dit uses.
258    ///
259    /// New-format page header (40 bytes):
260    /// ```text
261    ///   0..8    XOR checksum (u32 pair) + ECC checksum
262    ///   8..12   page number (or 0 for pre-Vista format where number is implicit)
263    ///  12..20   modified time (u64)
264    ///  20..24   previous page
265    ///  24..28   next page
266    ///  28..32   father data page
267    ///  32..34   available data size
268    ///  34..36   available uncommitted data size
269    ///  36..38   available data offset
270    ///  38..40   available page tag
271    ///  40..44   page flags
272    /// ```
273    ///
274    /// object_id sits at 44..48 in the new format's extended fields.
275    pub fn parse(page_number: u32, bytes: &[u8], page_size: u32) -> Result<Self, Error> {
276        if bytes.len() as u32 != page_size {
277            return Err(Error::Format(format!(
278                "page {page_number}: got {} bytes, expected {page_size}",
279                bytes.len()
280            )));
281        }
282        if page_size < 48 {
283            return Err(Error::Format(format!("page size too small: {page_size}")));
284        }
285        let previous_page = u32::from_le_bytes([bytes[20], bytes[21], bytes[22], bytes[23]]);
286        let next_page = u32::from_le_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]);
287        let father_data_page = u32::from_le_bytes([bytes[28], bytes[29], bytes[30], bytes[31]]);
288        let available_data_size = u16::from_le_bytes([bytes[32], bytes[33]]);
289        let available_page_tag = u16::from_le_bytes([bytes[38], bytes[39]]);
290        let page_flags = u32::from_le_bytes([bytes[40], bytes[41], bytes[42], bytes[43]]);
291        let object_id = u32::from_le_bytes([bytes[44], bytes[45], bytes[46], bytes[47]]);
292        Ok(RawPage {
293            page_number,
294            page_flags,
295            previous_page,
296            next_page,
297            father_data_page,
298            object_id,
299            available_data_size,
300            available_page_tag,
301            bytes: bytes.to_vec(),
302        })
303    }
304
305    /// True if the page has all of the specified [`page_flag`] bits set.
306    pub fn has_flag(&self, flag: u32) -> bool {
307        (self.page_flags & flag) == flag
308    }
309
310    /// True if this is a leaf page (holds actual data records).
311    pub fn is_leaf(&self) -> bool {
312        self.has_flag(page_flag::LEAF)
313    }
314
315    /// True if this is a root page.
316    pub fn is_root(&self) -> bool {
317        self.has_flag(page_flag::ROOT)
318    }
319}
320
321/// Errors this crate returns.
322#[derive(Debug)]
323#[non_exhaustive]
324pub enum Error {
325    Io(std::io::Error),
326    Format(String),
327}
328
329impl std::fmt::Display for Error {
330    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
331        match self {
332            Error::Io(e) => write!(f, "io: {e}"),
333            Error::Format(s) => write!(f, "format: {s}"),
334        }
335    }
336}
337
338impl std::error::Error for Error {}
339
340#[cfg(test)]
341mod tests {
342    use super::*;
343
344    fn synthetic_header(page_size: u32) -> Vec<u8> {
345        let mut hdr = vec![0u8; 668];
346        hdr[4..8].copy_from_slice(&ESE_FILE_SIGNATURE.to_le_bytes());
347        hdr[8..12].copy_from_slice(&620u32.to_le_bytes()); // format-version
348        hdr[12..16].copy_from_slice(&file_type::DATABASE.to_le_bytes());
349        hdr[236..240].copy_from_slice(&page_size.to_le_bytes());
350        hdr[240..244].copy_from_slice(&17u32.to_le_bytes()); // format-revision
351        hdr
352    }
353
354    #[test]
355    fn header_accepts_valid_synthetic() {
356        let hdr = synthetic_header(8192);
357        let h = EseFile::parse_header(&hdr).unwrap();
358        assert_eq!(h.page_size, 8192);
359        assert_eq!(h.format_version, 620);
360        assert_eq!(h.format_revision, 17);
361        assert_eq!(h.file_type, file_type::DATABASE);
362    }
363
364    #[test]
365    fn header_rejects_short() {
366        assert!(matches!(
367            EseFile::parse_header(&[0u8; 300]).unwrap_err(),
368            Error::Format(_)
369        ));
370    }
371
372    #[test]
373    fn header_rejects_bad_magic() {
374        let mut hdr = synthetic_header(8192);
375        hdr[4..8].copy_from_slice(&0xdeadbeefu32.to_le_bytes());
376        assert!(matches!(
377            EseFile::parse_header(&hdr).unwrap_err(),
378            Error::Format(_)
379        ));
380    }
381
382    #[test]
383    fn header_rejects_odd_page_size() {
384        let hdr = synthetic_header(1234);
385        assert!(matches!(
386            EseFile::parse_header(&hdr).unwrap_err(),
387            Error::Format(_)
388        ));
389    }
390
391    #[test]
392    fn page_flags_have_expected_bits() {
393        // Guard against a typo — the LEAF/ROOT bit values are load-bearing in every
394        // downstream B-tree walker.
395        assert_eq!(page_flag::ROOT, 0x1);
396        assert_eq!(page_flag::LEAF, 0x2);
397        assert_eq!(page_flag::PARENT, 0x4);
398        assert_eq!(page_flag::LONG_VALUE, 0x80);
399    }
400
401    #[test]
402    fn raw_page_parse_reads_header_fields() {
403        let mut bytes = vec![0u8; 4096];
404        bytes[20..24].copy_from_slice(&100u32.to_le_bytes()); // previous_page
405        bytes[24..28].copy_from_slice(&102u32.to_le_bytes()); // next_page
406        bytes[28..32].copy_from_slice(&2u32.to_le_bytes()); // father_data_page
407        bytes[32..34].copy_from_slice(&512u16.to_le_bytes()); // available_data_size
408        bytes[38..40].copy_from_slice(&5u16.to_le_bytes()); // available_page_tag
409        bytes[40..44]
410            .copy_from_slice(&(page_flag::LEAF | page_flag::NEW_RECORD_FORMAT).to_le_bytes());
411        bytes[44..48].copy_from_slice(&42u32.to_le_bytes()); // object_id
412        let page = RawPage::parse(101, &bytes, 4096).unwrap();
413        assert_eq!(page.page_number, 101);
414        assert_eq!(page.previous_page, 100);
415        assert_eq!(page.next_page, 102);
416        assert_eq!(page.available_data_size, 512);
417        assert_eq!(page.available_page_tag, 5);
418        assert_eq!(page.object_id, 42);
419        assert!(page.is_leaf());
420        assert!(!page.is_root());
421        assert!(page.has_flag(page_flag::NEW_RECORD_FORMAT));
422    }
423
424    #[test]
425    fn raw_page_rejects_size_mismatch() {
426        let bytes = vec![0u8; 100];
427        assert!(matches!(
428            RawPage::parse(1, &bytes, 4096).unwrap_err(),
429            Error::Format(_)
430        ));
431    }
432}
ese_parser/lib.rs

ese_parser/
lib.rs
