ese-parser 0.1.0

//! **ese-parser** — Microsoft Extensible Storage Engine (ESE / "Jet Blue") file parser
//! in pure Rust.
//!
//! ESE is the on-disk B-tree database Windows uses for anything that needs local persistence
//! with recovery: `ntds.dit` (Active Directory), `WebCacheV*.dat` (Edge/IE history),
//! `Windows.edb` (Search index), `Mail.dat` (legacy Exchange), event-tracing logs.
//!
//! This crate is a **from-scratch** parser (no C-lib FFI) focused on read-only forensic /
//! offensive-security use — the subset a defender or attacker needs to extract data from a
//! captured `.dit`/`.edb`, not the full writable Jet API.
//!
//! # v0.1 scope
//!
//! - [`EseFile::open`] — parse the 668-byte database header (magic, format-version,
//!   page-size, file-type, previous-full-backup / previous-incremental-backup metadata,
//!   the min/max log-generation numbers required to know if the DB is consistent).
//! - [`EseFile::page`] — random-access read of a specific page by page number, returning a
//!   [`RawPage`] with parsed header fields (page number, previous/next page in the B-tree
//!   sibling chain, page flags telling you whether it's a leaf/branch/root/parent-of-leaf).
//! - [`RawPage`] constants — [`page_flag`] bits, [`tag`] flag bits.
//!
//! # v0.2 roadmap
//!
//! - B-tree walk (root → branch → leaf, following child page pointers).
//! - Catalog table decode — the metadata table that lists every user table + its column
//!   definitions.
//! - Row / tag decode with long-value + multi-value reassembly.
//! - Then: this becomes the backend for
//!   [`ntds-parse`](https://crates.io/crates/ntds-parse) `EseBackend` trait so callers can
//!   `for row in ntds.datatable()`.
//!
//! # Non-goals
//!
//! - Writable Jet API (this is read-only).
//! - Recovery from a dirty shutdown (defer that to Windows' own `esentutl /p`).
//! - Encrypted databases (Windows database encryption uses per-page XOR with a table key —
//!   out of scope for v0.1).
//!
//! # Verification
//!
//! Header offsets validated against
//! [libesedb's spec doc](https://github.com/libyal/libesedb/blob/main/documentation/Extensible%20Storage%20Engine%20(ESE)%20Database%20File%20(EDB)%20format.asciidoc).
//! Page header layout matches Joachim Metz's reverse-engineering notes.

use std::fs::File;
use std::io::{Read, Seek, SeekFrom};
use std::path::Path;

/// The 4-byte little-endian file signature every ESE database starts with (offset 4).
pub const ESE_FILE_SIGNATURE: u32 = 0x89AB_CDEF;

/// File-type values in the database header (offset 12).
pub mod file_type {
    pub const DATABASE: u32 = 0;
    pub const STREAMING_FILE: u32 = 1;
}

/// Page-header flag bits ([MS-ESE] and libesedb reference).
pub mod page_flag {
    /// Page is a root page of a B-tree.
    pub const ROOT: u32 = 0x0000_0001;
    /// Page is a leaf page (holds data records).
    pub const LEAF: u32 = 0x0000_0002;
    /// Page is a parent-of-leaf (its child pages are all leaves).
    pub const PARENT: u32 = 0x0000_0004;
    /// Page is empty (fresh allocation).
    pub const EMPTY: u32 = 0x0000_0008;
    /// Page contains a space tree node.
    pub const SPACE_TREE: u32 = 0x0000_0020;
    /// Page belongs to an index B-tree (not the primary/data tree).
    pub const INDEX: u32 = 0x0000_0040;
    /// Page is part of a long-value chunk chain.
    pub const LONG_VALUE: u32 = 0x0000_0080;
    /// Page has been written to since the last full-database backup.
    pub const NEW_RECORD_FORMAT: u32 = 0x0000_2000;
    /// Page has scrubbed data (deleted rows overwritten).
    pub const SCRUBBED: u32 = 0x0000_4000;
    /// Page has a new checksum format (post-Windows Vista).
    pub const NEW_CHECKSUM_FORMAT: u32 = 0x0000_8000;
}

/// A minimally-parsed ESE database file — reads the header + gives you random-access page
/// reads. Does not walk B-trees or enumerate tables (v0.2 roadmap).
pub struct EseFile {
    file: File,
    format_version: u32,
    format_revision: u32,
    page_size: u32,
    file_type: u32,
    previous_incremental_backup_log_gen: u32,
    previous_full_backup_log_gen: u32,
    current_full_backup_log_gen: u32,
}

impl EseFile {
    /// Open + validate an ESE database file. Reads only the 668-byte database header at
    /// open-time; page reads happen on demand.
    pub fn open(path: impl AsRef<Path>) -> Result<Self, Error> {
        let mut file = File::open(path).map_err(Error::Io)?;
        let mut hdr = [0u8; 668];
        file.read_exact(&mut hdr).map_err(Error::Io)?;
        let parsed = Self::parse_header_fields(&hdr)?;
        Ok(EseFile {
            file,
            format_version: parsed.format_version,
            format_revision: parsed.format_revision,
            page_size: parsed.page_size,
            file_type: parsed.file_type,
            previous_incremental_backup_log_gen: parsed.previous_incremental_backup_log_gen,
            previous_full_backup_log_gen: parsed.previous_full_backup_log_gen,
            current_full_backup_log_gen: parsed.current_full_backup_log_gen,
        })
    }

    /// Parse the 668-byte header without any I/O. Useful for tests or when the caller has
    /// already read the bytes over the wire.
    pub fn parse_header(hdr: &[u8]) -> Result<HeaderFields, Error> {
        Self::parse_header_fields(hdr)
    }

    fn parse_header_fields(hdr: &[u8]) -> Result<HeaderFields, Error> {
        if hdr.len() < 668 {
            return Err(Error::Format(format!(
                "header too short: {} bytes (want 668)",
                hdr.len()
            )));
        }
        let magic = u32::from_le_bytes([hdr[4], hdr[5], hdr[6], hdr[7]]);
        if magic != ESE_FILE_SIGNATURE {
            return Err(Error::Format(format!(
                "bad ESE magic: got {magic:#010x}, want {ESE_FILE_SIGNATURE:#010x}"
            )));
        }
        let format_version = u32::from_le_bytes([hdr[8], hdr[9], hdr[10], hdr[11]]);
        let file_type = u32::from_le_bytes([hdr[12], hdr[13], hdr[14], hdr[15]]);
        let page_size = u32::from_le_bytes([hdr[236], hdr[237], hdr[238], hdr[239]]);
        if ![2048, 4096, 8192, 16384, 32768].contains(&page_size) {
            return Err(Error::Format(format!("odd page size: {page_size}")));
        }
        let format_revision = u32::from_le_bytes([hdr[240], hdr[241], hdr[242], hdr[243]]);
        // Backup log-gen fields — used to detect a dirty shutdown, the caller can decide
        // whether to bail on inconsistent files.
        // Offsets per libesedb spec:
        //   248..260 = previous full backup snapshot
        //   260..272 = previous incremental backup snapshot
        //   272..284 = current full backup snapshot
        // Each snapshot is a LOG_POSITION struct (8-byte LSN + 4-byte log-gen); the log-gen
        // is at snapshot base + 8.
        let previous_full_backup_log_gen =
            u32::from_le_bytes([hdr[256], hdr[257], hdr[258], hdr[259]]);
        let previous_incremental_backup_log_gen =
            u32::from_le_bytes([hdr[268], hdr[269], hdr[270], hdr[271]]);
        let current_full_backup_log_gen =
            u32::from_le_bytes([hdr[280], hdr[281], hdr[282], hdr[283]]);
        Ok(HeaderFields {
            format_version,
            format_revision,
            page_size,
            file_type,
            previous_full_backup_log_gen,
            previous_incremental_backup_log_gen,
            current_full_backup_log_gen,
        })
    }

    pub fn format_version(&self) -> u32 {
        self.format_version
    }
    pub fn format_revision(&self) -> u32 {
        self.format_revision
    }
    pub fn page_size(&self) -> u32 {
        self.page_size
    }
    pub fn file_type(&self) -> u32 {
        self.file_type
    }
    pub fn previous_full_backup_log_gen(&self) -> u32 {
        self.previous_full_backup_log_gen
    }
    pub fn previous_incremental_backup_log_gen(&self) -> u32 {
        self.previous_incremental_backup_log_gen
    }
    pub fn current_full_backup_log_gen(&self) -> u32 {
        self.current_full_backup_log_gen
    }

    /// Read a page from the database by 1-indexed page number (page 0 is the header, page 1
    /// is the shadow header, real pages start at 2). Returns the parsed [`RawPage`].
    pub fn page(&mut self, page_number: u32) -> Result<RawPage, Error> {
        let offset = page_number as u64 * self.page_size as u64;
        self.file.seek(SeekFrom::Start(offset)).map_err(Error::Io)?;
        let mut buf = vec![0u8; self.page_size as usize];
        self.file.read_exact(&mut buf).map_err(Error::Io)?;
        RawPage::parse(page_number, &buf, self.page_size)
    }
}

/// Result of parsing the 668-byte header without holding onto the file. Callers use this
/// when they want to display + validate header metadata without keeping an open handle.
#[derive(Debug, Clone, Copy)]
#[non_exhaustive]
pub struct HeaderFields {
    pub format_version: u32,
    pub format_revision: u32,
    pub page_size: u32,
    pub file_type: u32,
    pub previous_full_backup_log_gen: u32,
    pub previous_incremental_backup_log_gen: u32,
    pub current_full_backup_log_gen: u32,
}

/// One page read out of the ESE database. Header parsed; page-body bytes retained so
/// downstream (a future B-tree walker) can decode tags on demand.
#[derive(Clone)]
#[non_exhaustive]
pub struct RawPage {
    /// 1-indexed page number.
    pub page_number: u32,
    /// Bitmask of [`page_flag`] flags.
    pub page_flags: u32,
    /// Page number of the previous sibling in the B-tree chain (0 = none).
    pub previous_page: u32,
    /// Page number of the next sibling in the B-tree chain (0 = none).
    pub next_page: u32,
    /// Father Data Page (the parent B-tree node's page number).
    pub father_data_page: u32,
    /// Object identifier this page belongs to (the FDP object ID).
    pub object_id: u32,
    /// Available data size (bytes free for new tags on this page).
    pub available_data_size: u16,
    /// Number of tag entries on this page.
    pub available_page_tag: u16,
    /// The full page bytes (retained so B-tree/tag decoders can work on top).
    pub bytes: Vec<u8>,
}

impl std::fmt::Debug for RawPage {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("RawPage")
            .field("page_number", &self.page_number)
            .field("page_flags", &format_args!("{:#010x}", self.page_flags))
            .field("previous_page", &self.previous_page)
            .field("next_page", &self.next_page)
            .field("father_data_page", &self.father_data_page)
            .field("object_id", &self.object_id)
            .field("available_data_size", &self.available_data_size)
            .field("available_page_tag", &self.available_page_tag)
            .field("bytes_len", &self.bytes.len())
            .finish()
    }
}

impl RawPage {
    /// Parse a page's header. The layout differs between the "old" (pre-Vista, page-size-
    /// specific) and "new" (post-Vista, standard 40-byte header) formats. v0.1 supports the
    /// new format that Server 2008+ NTDS.dit uses.
    ///
    /// New-format page header (40 bytes):
    /// ```text
    ///   0..8    XOR checksum (u32 pair) + ECC checksum
    ///   8..12   page number (or 0 for pre-Vista format where number is implicit)
    ///  12..20   modified time (u64)
    ///  20..24   previous page
    ///  24..28   next page
    ///  28..32   father data page
    ///  32..34   available data size
    ///  34..36   available uncommitted data size
    ///  36..38   available data offset
    ///  38..40   available page tag
    ///  40..44   page flags
    /// ```
    ///
    /// object_id sits at 44..48 in the new format's extended fields.
    pub fn parse(page_number: u32, bytes: &[u8], page_size: u32) -> Result<Self, Error> {
        if bytes.len() as u32 != page_size {
            return Err(Error::Format(format!(
                "page {page_number}: got {} bytes, expected {page_size}",
                bytes.len()
            )));
        }
        if page_size < 48 {
            return Err(Error::Format(format!("page size too small: {page_size}")));
        }
        let previous_page = u32::from_le_bytes([bytes[20], bytes[21], bytes[22], bytes[23]]);
        let next_page = u32::from_le_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]);
        let father_data_page = u32::from_le_bytes([bytes[28], bytes[29], bytes[30], bytes[31]]);
        let available_data_size = u16::from_le_bytes([bytes[32], bytes[33]]);
        let available_page_tag = u16::from_le_bytes([bytes[38], bytes[39]]);
        let page_flags = u32::from_le_bytes([bytes[40], bytes[41], bytes[42], bytes[43]]);
        let object_id = u32::from_le_bytes([bytes[44], bytes[45], bytes[46], bytes[47]]);
        Ok(RawPage {
            page_number,
            page_flags,
            previous_page,
            next_page,
            father_data_page,
            object_id,
            available_data_size,
            available_page_tag,
            bytes: bytes.to_vec(),
        })
    }

    /// True if the page has all of the specified [`page_flag`] bits set.
    pub fn has_flag(&self, flag: u32) -> bool {
        (self.page_flags & flag) == flag
    }

    /// True if this is a leaf page (holds actual data records).
    pub fn is_leaf(&self) -> bool {
        self.has_flag(page_flag::LEAF)
    }

    /// True if this is a root page.
    pub fn is_root(&self) -> bool {
        self.has_flag(page_flag::ROOT)
    }
}

/// Errors this crate returns.
#[derive(Debug)]
#[non_exhaustive]
pub enum Error {
    Io(std::io::Error),
    Format(String),
}

impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Error::Io(e) => write!(f, "io: {e}"),
            Error::Format(s) => write!(f, "format: {s}"),
        }
    }
}

impl std::error::Error for Error {}

#[cfg(test)]
mod tests {
    use super::*;

    fn synthetic_header(page_size: u32) -> Vec<u8> {
        let mut hdr = vec![0u8; 668];
        hdr[4..8].copy_from_slice(&ESE_FILE_SIGNATURE.to_le_bytes());
        hdr[8..12].copy_from_slice(&620u32.to_le_bytes()); // format-version
        hdr[12..16].copy_from_slice(&file_type::DATABASE.to_le_bytes());
        hdr[236..240].copy_from_slice(&page_size.to_le_bytes());
        hdr[240..244].copy_from_slice(&17u32.to_le_bytes()); // format-revision
        hdr
    }

    #[test]
    fn header_accepts_valid_synthetic() {
        let hdr = synthetic_header(8192);
        let h = EseFile::parse_header(&hdr).unwrap();
        assert_eq!(h.page_size, 8192);
        assert_eq!(h.format_version, 620);
        assert_eq!(h.format_revision, 17);
        assert_eq!(h.file_type, file_type::DATABASE);
    }

    #[test]
    fn header_rejects_short() {
        assert!(matches!(
            EseFile::parse_header(&[0u8; 300]).unwrap_err(),
            Error::Format(_)
        ));
    }

    #[test]
    fn header_rejects_bad_magic() {
        let mut hdr = synthetic_header(8192);
        hdr[4..8].copy_from_slice(&0xdeadbeefu32.to_le_bytes());
        assert!(matches!(
            EseFile::parse_header(&hdr).unwrap_err(),
            Error::Format(_)
        ));
    }

    #[test]
    fn header_rejects_odd_page_size() {
        let hdr = synthetic_header(1234);
        assert!(matches!(
            EseFile::parse_header(&hdr).unwrap_err(),
            Error::Format(_)
        ));
    }

    #[test]
    fn page_flags_have_expected_bits() {
        // Guard against a typo — the LEAF/ROOT bit values are load-bearing in every
        // downstream B-tree walker.
        assert_eq!(page_flag::ROOT, 0x1);
        assert_eq!(page_flag::LEAF, 0x2);
        assert_eq!(page_flag::PARENT, 0x4);
        assert_eq!(page_flag::LONG_VALUE, 0x80);
    }

    #[test]
    fn raw_page_parse_reads_header_fields() {
        let mut bytes = vec![0u8; 4096];
        bytes[20..24].copy_from_slice(&100u32.to_le_bytes()); // previous_page
        bytes[24..28].copy_from_slice(&102u32.to_le_bytes()); // next_page
        bytes[28..32].copy_from_slice(&2u32.to_le_bytes()); // father_data_page
        bytes[32..34].copy_from_slice(&512u16.to_le_bytes()); // available_data_size
        bytes[38..40].copy_from_slice(&5u16.to_le_bytes()); // available_page_tag
        bytes[40..44]
            .copy_from_slice(&(page_flag::LEAF | page_flag::NEW_RECORD_FORMAT).to_le_bytes());
        bytes[44..48].copy_from_slice(&42u32.to_le_bytes()); // object_id
        let page = RawPage::parse(101, &bytes, 4096).unwrap();
        assert_eq!(page.page_number, 101);
        assert_eq!(page.previous_page, 100);
        assert_eq!(page.next_page, 102);
        assert_eq!(page.available_data_size, 512);
        assert_eq!(page.available_page_tag, 5);
        assert_eq!(page.object_id, 42);
        assert!(page.is_leaf());
        assert!(!page.is_root());
        assert!(page.has_flag(page_flag::NEW_RECORD_FORMAT));
    }

    #[test]
    fn raw_page_rejects_size_mismatch() {
        let bytes = vec![0u8; 100];
        assert!(matches!(
            RawPage::parse(1, &bytes, 4096).unwrap_err(),
            Error::Format(_)
        ));
    }
}