readmdict 0.1.0

//! Core readmdict implementation
//! Direct port of readmdict/readmdict/readmdict.py

use std::collections::HashMap;
use std::fs::File;
use std::io::{Read, Seek, SeekFrom, Cursor};
use byteorder::{BigEndian, LittleEndian, ReadBytesExt};

use salsa20::Salsa20;
use salsa20::cipher::{KeyIvInit, StreamCipher};
use ripemd::{Ripemd128, Digest};
use adler::Adler32;
// struct_pack functionality will be implemented manually when needed

// Error types
#[derive(Debug, thiserror::Error)]
pub enum Error {
    #[error("IO error: {0}")]
    Io(#[from] std::io::Error),
    #[error("Invalid file format: {0}")]
    InvalidFormat(String),
    #[error("Unsupported compression type")]
    UnsupportedCompression,
    #[error("Encryption error: {0}")]
    Encryption(String),
    #[error("Invalid passcode")]
    InvalidPasscode,
    #[error("Checksum mismatch")]
    ChecksumMismatch,
    #[error("Encoding error: {0}")]
    Encoding(String),
    #[error("Parse error: {0}")]
    Parse(String),
}

pub type Result<T> = std::result::Result<T, Error>;

// Passcode struct
// Corresponds to passcode parsing in readmdict.py:607-616
#[derive(Debug, Clone)]
pub struct Passcode {
    pub regcode: Vec<u8>,
    pub userid: String,
}

// Utility functions - direct ports from Python

/// Convert HTML entities back to characters
/// Port of _unescape_entities from readmdict.py:43-50
fn unescape_entities(text: &[u8]) -> Vec<u8> {
    let mut result = text.to_vec();
    
    // Replace HTML entities
    let replacements = [
        (b"&lt;".as_slice(), b"<".as_slice()),
        (b"&gt;", b">"),
        (b"&amp;", b"&"),
        (b"&quot;", b"\""),
    ];
    
    for (entity, replacement) in &replacements {
        let mut pos = 0;
        while let Some(found) = result[pos..].windows(entity.len()).position(|w| w == *entity) {
            let actual_pos = pos + found;
            result.splice(actual_pos..actual_pos + entity.len(), replacement.iter().cloned());
            pos = actual_pos + replacement.len();
        }
    }
    
    result
}

/// Simple XOR decryption with key cycling
/// Port of _fast_decrypt from readmdict.py:52-62
fn fast_decrypt(data: &[u8], key: &[u8]) -> Vec<u8> {
    if key.is_empty() {
        return data.to_vec();
    }
    
    data.iter()
        .enumerate()
        .map(|(i, &byte)| byte ^ key[i % key.len()])
        .collect()
}

/// MDX-specific decryption algorithm
/// Port of _mdx_decrypt from readmdict.py:64-67
fn mdx_decrypt(comp_block: &[u8]) -> Result<Vec<u8>> {
    let key = b"\x95\x36\x00\x00";
    Ok(fast_decrypt(comp_block, key))
}

/// Salsa20 decryption using external crate
/// Port of _salsa_decrypt from readmdict.py:69-72
fn salsa_decrypt(ciphertext: &[u8], key: &[u8]) -> Result<Vec<u8>> {
    if key.len() != 32 {
        return Err(Error::Encryption("Salsa20 key must be 32 bytes".to_string()));
    }
    
    let mut cipher = Salsa20::new_from_slices(key, &[0u8; 8])
        .map_err(|e| Error::Encryption(format!("Salsa20 init error: {}", e)))?;
    
    let mut result = ciphertext.to_vec();
    cipher.apply_keystream(&mut result);
    Ok(result)
}

/// Device ID based decryption
/// Port of _decrypt_regcode_by_deviceid from readmdict.py:74-79
fn decrypt_regcode_by_deviceid(regcode: &[u8], deviceid: &[u8]) -> Result<Vec<u8>> {
    let mut hasher = Ripemd128::new();
    hasher.update(deviceid);
    hasher.update(regcode);
    let digest = hasher.finalize();
    
    let mut key = [0u8; 32];
    key[..16].copy_from_slice(&digest);
    key[16..].copy_from_slice(&digest);
    
    salsa_decrypt(regcode, &key)
}

/// Email based decryption
/// Port of _decrypt_regcode_by_email from readmdict.py:81-86
fn decrypt_regcode_by_email(regcode: &[u8], email: &[u8]) -> Result<Vec<u8>> {
    let mut hasher = Ripemd128::new();
    hasher.update(email);
    hasher.update(regcode);
    let digest = hasher.finalize();
    
    let mut key = [0u8; 32];
    key[..16].copy_from_slice(&digest);
    key[16..].copy_from_slice(&digest);
    
    salsa_decrypt(regcode, &key)
}

// Base MDict struct - equivalent to Python MDict class
// Port of MDict class from readmdict.py:88-404
#[derive(Debug)]
pub struct MDict {
    fname: String,
    encoding: String,
    passcode: Option<Passcode>,
    header: HashMap<String, String>,
    key_list: Vec<(u64, Vec<u8>)>,
    num_entries: usize,
    version: f32,
    encrypt: u8,
    number_width: usize,
    key_block_offset: u64,
    record_block_offset: u64,
    stylesheet: HashMap<String, (String, String)>,
}

impl MDict {
    /// Constructor - direct port of Python MDict.__init__
    /// Port of __init__ from readmdict.py:95-106
    pub fn new(fname: &str, encoding: Option<String>, passcode: Option<Passcode>) -> Result<Self> {
        let mut mdict = Self {
            fname: fname.to_string(),
            encoding: encoding.unwrap_or_default(),
            passcode,
            header: HashMap::new(),
            key_list: Vec::new(),
            num_entries: 0,
            version: 0.0,
            encrypt: 0,
            number_width: 4,
            key_block_offset: 0,
            record_block_offset: 0,
            stylesheet: HashMap::new(),
        };
        
        // Read header and keys - matches readmdict.py:100-105
        mdict.header = mdict.read_header()?;
        mdict.key_list = match mdict.read_keys() {
            Ok(keys) => keys,
            Err(_) => {
                println!("Try Brutal Force on Encrypted Key Blocks");
                mdict.read_keys_brutal()?
            }
        };
        
        Ok(mdict)
    }
    
    /// Length - direct port of Python MDict.__len__
    /// Port of __len__ from readmdict.py:108-109
    pub fn len(&self) -> usize {
        self.num_entries
    }
    
    /// Check if empty
    pub fn is_empty(&self) -> bool {
        self.num_entries == 0
    }
    
    /// Keys iterator - direct port of Python MDict.keys
    /// Port of keys from readmdict.py:114-116
    pub fn keys(&self) -> impl Iterator<Item = &[u8]> {
        self.key_list.iter().map(|(_, key)| key.as_slice())
    }
    
    /// Get header reference
    pub fn header(&self) -> &HashMap<String, String> {
        &self.header
    }
    
    /// Read number based on version (4 or 8 bytes)
    /// Port of _read_number from readmdict.py:118-119
    fn read_number<R: Read>(&self, reader: &mut R) -> Result<u64> {
        if self.number_width == 4 {
            Ok(reader.read_u32::<BigEndian>()? as u64)
        } else {
            Ok(reader.read_u64::<BigEndian>()?)
        }
    }
    
    /// Parse XML-like header attributes
    /// Port of _parse_header from readmdict.py:121-127
    fn parse_header(header: &[u8]) -> Result<HashMap<String, String>> {
        let mut tagdict = HashMap::new();
        let header_str = String::from_utf8_lossy(header);
        
        // Manual parsing for key="value" patterns
        let mut chars = header_str.chars().peekable();
        while let Some(ch) = chars.next() {
            if ch.is_alphabetic() {
                // Parse key
                let mut key = String::new();
                key.push(ch);
                while let Some(&next_ch) = chars.peek() {
                    if next_ch.is_alphanumeric() || next_ch == '_' {
                        key.push(chars.next().unwrap());
                    } else {
                        break;
                    }
                }
                
                // Skip whitespace and '='
                while let Some(&next_ch) = chars.peek() {
                    if next_ch.is_whitespace() || next_ch == '=' {
                        chars.next();
                    } else {
                        break;
                    }
                }
                
                // Parse quoted value
                if let Some('"') = chars.next() {
                    let mut value = String::new();
                    while let Some(ch) = chars.next() {
                        if ch == '"' {
                            break;
                        }
                        value.push(ch);
                    }
                    
                    let value_bytes = unescape_entities(value.as_bytes());
                    let final_value = String::from_utf8_lossy(&value_bytes).to_string();
                    tagdict.insert(key, final_value);
                }
            }
        }
        
        Ok(tagdict)
    }
    
    /// Read and parse file header
    /// Port of _read_header from readmdict.py:244-301
    fn read_header(&mut self) -> Result<HashMap<String, String>> {
        let mut f = File::open(&self.fname)?;
        
        // Read header size
        let header_bytes_size = f.read_u32::<BigEndian>()? as usize;
        let mut header_bytes = vec![0u8; header_bytes_size];
        f.read_exact(&mut header_bytes)?;
        
        // Read and verify checksum
        let adler32_expected = f.read_u32::<LittleEndian>()?;
        let mut hasher = Adler32::new();
        hasher.write_slice(&header_bytes);
        let adler32_actual = hasher.checksum();
        if adler32_expected != adler32_actual {
            return Err(Error::ChecksumMismatch);
        }
        
        // Mark key block offset
        self.key_block_offset = f.stream_position()?;
        
        // Parse header text (UTF-16 ending with \x00\x00)
        let header_text = if header_bytes.len() >= 2 {
            let text_bytes = &header_bytes[..header_bytes.len() - 2];
            
            // Try UTF-16LE decoding first
            let decoded = if text_bytes.len() % 2 == 0 {
                let utf16: Vec<u16> = text_bytes
                    .chunks_exact(2)
                    .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
                    .collect();
                String::from_utf16(&utf16).unwrap_or_else(|_| String::from_utf8_lossy(text_bytes).into_owned())
            } else {
                String::from_utf8_lossy(text_bytes).into_owned()
            };
            
            decoded.into_bytes()
        } else {
            return Err(Error::InvalidFormat("Header too short".to_string()));
        };
        
        let header_tag = Self::parse_header(&header_text)?;
        
        // Set encoding if not provided
        if self.encoding.is_empty() {
            if let Some(encoding) = header_tag.get("Encoding") {
                let mut enc = encoding.clone();
                // GB18030 > GBK > GB2312
                if enc == "GBK" || enc == "GB2312" {
                    enc = "GB18030".to_string();
                }
                self.encoding = enc;
            }
        }
        
        // Set encryption flag
        self.encrypt = if let Some(encrypted) = header_tag.get("Encrypted") {
            match encrypted.as_str() {
                "No" => 0,
                "Yes" => 1,
                _ => encrypted.parse().unwrap_or(0),
            }
        } else {
            0
        };
        
        // Parse stylesheet
        if let Some(stylesheet_str) = header_tag.get("StyleSheet") {
            let lines: Vec<&str> = stylesheet_str.lines().collect();
            for chunk in lines.chunks(3) {
                if chunk.len() == 3 {
                    self.stylesheet.insert(
                        chunk[0].to_string(),
                        (chunk[1].to_string(), chunk[2].to_string()),
                    );
                }
            }
        }
        
        // Set version and number format
        if let Some(version_str) = header_tag.get("RequiredEngineVersion") {
            self.version = version_str.parse().unwrap_or(1.0);
            if self.version < 2.0 {
                self.number_width = 4;
            } else {
                self.number_width = 8;
            }
        } else if let Some(version_str) = header_tag.get("GeneratedByEngineVersion") {
            self.version = version_str.parse().unwrap_or(1.0);
            if self.version < 2.0 {
                self.number_width = 4;
            } else {
                self.number_width = 8;
            }
        } else {
            self.version = 1.0;
            self.number_width = 4;
        }
        
        Ok(header_tag)
    }
    
    /// Read key blocks from file
    /// Port of _read_keys from readmdict.py:302-358
    fn read_keys(&mut self) -> Result<Vec<(u64, Vec<u8>)>> {
        let mut f = File::open(&self.fname)?;
        f.seek(SeekFrom::Start(self.key_block_offset))?;

        // The following numbers could be encrypted
        let num_bytes = if self.version >= 2.0 {
            8 * 5  // 5 numbers, 8 bytes each
        } else {
            4 * 4  // 4 numbers, 4 bytes each
        };
        
        let mut block = vec![0u8; num_bytes];
        f.read_exact(&mut block)?;

        // Decrypt if encrypted
        if self.encrypt & 1 != 0 {
            if let Some(ref passcode) = self.passcode {
                let userid = passcode.userid.as_bytes();
                let regcode = &passcode.regcode;
                
                let encrypted_key = if let Some(register_by) = self.header.get("RegisterBy") {
                    if register_by == "EMail" {
                        decrypt_regcode_by_email(regcode, userid)?
                    } else {
                        decrypt_regcode_by_deviceid(regcode, userid)?
                    }
                } else {
                    decrypt_regcode_by_deviceid(regcode, userid)?
                };
                
                block = salsa_decrypt(&block, &encrypted_key)?;
            } else {
                return Err(Error::InvalidPasscode);
            }
        }

        // Decode this block
        let mut cursor = Cursor::new(&block);
        
        // Number of key blocks
        let num_key_blocks = self.read_number(&mut cursor)?;
        
        // Number of entries
        self.num_entries = self.read_number(&mut cursor)? as usize;
        
        // Number of bytes of key block info after decompression (v2.0+)
        let _key_block_info_decomp_size = if self.version >= 2.0 {
            self.read_number(&mut cursor)?
        } else {
            0
        };
        
        // Number of bytes of key block info
        let key_block_info_size = self.read_number(&mut cursor)? as usize;
        
        // Number of bytes of key block
        let key_block_size = self.read_number(&mut cursor)? as usize;

        // 4 bytes: adler checksum of previous numbers (v2.0+)
        if self.version >= 2.0 {
            let adler32_expected = f.read_u32::<BigEndian>()?;
            let mut hasher = Adler32::new();
            hasher.write_slice(&block);
            let adler32_actual = hasher.checksum();
            if adler32_expected != adler32_actual {
                return Err(Error::ChecksumMismatch);
            }
        }

        // record block info section
        let mut key_block_info = vec![0u8; key_block_info_size];
        f.read_exact(&mut key_block_info)?;
        let key_block_info_list = self.decode_key_block_info(&key_block_info)?;

        if num_key_blocks as usize != key_block_info_list.len() {
            return Err(Error::InvalidFormat("Key block count mismatch".to_string()));
        }

        // actual key block data
        let mut key_block_compressed = vec![0u8; key_block_size];
        f.read_exact(&mut key_block_compressed)?;
        
        // extract key block
        let key_list = self.decode_key_block(&key_block_compressed, &key_block_info_list)?;

        self.record_block_offset = f.stream_position()?;
        
        Ok(key_list)
    }
    
    /// Read key blocks using brutal force method for encrypted files
    /// Port of _read_keys_brutal from readmdict.py:359-404
    fn read_keys_brutal(&mut self) -> Result<Vec<(u64, Vec<u8>)>> {
        let mut f = File::open(&self.fname)?;
        f.seek(SeekFrom::Start(self.key_block_offset))?;

        // The following numbers could be encrypted, disregard them!
        let (num_bytes, key_block_type) = if self.version >= 2.0 {
            (8 * 5 + 4, b"\x02\x00\x00\x00".to_vec())
        } else {
            (4 * 4, b"\x01\x00\x00\x00".to_vec())
        };
        
        let mut _block = vec![0u8; num_bytes];
        f.read_exact(&mut _block)?;

        // Key block info
        // 4 bytes '\x02\x00\x00\x00' (or '\x01\x00\x00\x00' for v1)
        // 4 bytes adler32 checksum
        // Unknown number of bytes follows until key_block_type which marks the beginning of key block
        let mut key_block_info = vec![0u8; 8];
        f.read_exact(&mut key_block_info)?;
        
        if self.version >= 2.0 {
            if &key_block_info[..4] != b"\x02\x00\x00\x00" {
                return Err(Error::InvalidFormat("Invalid key block marker".to_string()));
            }
        }

        // Read until we find the key block type marker
        loop {
            let fpos = f.stream_position()?;
            let mut buffer = vec![0u8; 1024];
            let bytes_read = f.read(&mut buffer)?;
            
            if bytes_read == 0 {
                return Err(Error::InvalidFormat("Key block type marker not found".to_string()));
            }
            
            buffer.truncate(bytes_read);
            
            if let Some(index) = buffer.windows(key_block_type.len())
                .position(|window| window == key_block_type) {
                key_block_info.extend_from_slice(&buffer[..index]);
                f.seek(SeekFrom::Start(fpos + index as u64))?;
                break;
            } else {
                key_block_info.extend_from_slice(&buffer);
            }
        }

        let key_block_info_list = self.decode_key_block_info(&key_block_info)?;
        let key_block_size: usize = key_block_info_list.iter().map(|(size, _)| *size as usize).sum();

        // Read key block
        let mut key_block_compressed = vec![0u8; key_block_size];
        f.read_exact(&mut key_block_compressed)?;
        
        // Extract key block
        let key_list = self.decode_key_block(&key_block_compressed, &key_block_info_list)?;

        self.record_block_offset = f.stream_position()?;
        
        Ok(key_list)
    }
    
    /// Decode key block info
    /// Port of _decode_key_block_info from readmdict.py:129-188
    fn decode_key_block_info(&self, key_block_info_compressed: &[u8]) -> Result<Vec<(u64, u64)>> {
        let key_block_info = if self.version >= 2.0 {
            // zlib compression
            if key_block_info_compressed.len() < 4 || &key_block_info_compressed[0..4] != b"\x02\x00\x00\x00" {
                return Err(Error::Parse("Invalid key block info header".to_string()));
            }
            
            let mut data = key_block_info_compressed;
            let decrypted_data;
            // decrypt if needed
            if self.encrypt & 0x02 != 0 {
                decrypted_data = mdx_decrypt(data)?;
                data = &decrypted_data;
            }
            
            // decompress
            let mut decoder = flate2::read::ZlibDecoder::new(&data[8..]);
            let mut key_block_info = Vec::new();
            std::io::Read::read_to_end(&mut decoder, &mut key_block_info)
                .map_err(|e| Error::Io(e))?;
            
            // adler checksum
            let adler32_expected = u32::from_be_bytes([
                data[4], data[5], data[6], data[7]
            ]);
            let mut hasher = Adler32::new();
            hasher.write_slice(&key_block_info);
            let adler32_actual = hasher.checksum();
            if adler32_expected != adler32_actual {
                return Err(Error::Parse("Adler32 checksum mismatch".to_string()));
            }
            
            key_block_info
        } else {
            // no compression
            key_block_info_compressed.to_vec()
        };
        
        // decode
        let mut key_block_info_list = Vec::new();
        let mut _num_entries = 0u64;
        let mut i = 0;
        
        let (byte_format_width, text_term) = if self.version >= 2.0 {
            (2, 1) // >H format, 2 bytes width, 1 text terminator
        } else {
            (1, 0) // >B format, 1 byte width, 0 text terminator
        };
        
        while i < key_block_info.len() {
            if i + self.number_width > key_block_info.len() {
                break;
            }
            
            // number of entries in current key block
            let entries = self.read_number_from_bytes(&key_block_info[i..i + self.number_width])?;
            _num_entries += entries;
            i += self.number_width;
            
            if i + byte_format_width > key_block_info.len() {
                break;
            }
            
            // text head size
            let text_head_size = if byte_format_width == 2 {
                u16::from_be_bytes([key_block_info[i], key_block_info[i + 1]]) as usize
            } else {
                key_block_info[i] as usize
            };
            i += byte_format_width;
            
            // text head
            let head_skip = if self.encoding != "UTF-16" {
                text_head_size + text_term
            } else {
                (text_head_size + text_term) * 2
            };
            i += head_skip;
            
            if i + byte_format_width > key_block_info.len() {
                break;
            }
            
            // text tail size
            let text_tail_size = if byte_format_width == 2 {
                u16::from_be_bytes([key_block_info[i], key_block_info[i + 1]]) as usize
            } else {
                key_block_info[i] as usize
            };
            i += byte_format_width;
            
            // text tail
            let tail_skip = if self.encoding != "UTF-16" {
                text_tail_size + text_term
            } else {
                (text_tail_size + text_term) * 2
            };
            i += tail_skip;
            
            if i + 2 * self.number_width > key_block_info.len() {
                break;
            }
            
            // key block compressed size
            let key_block_compressed_size = self.read_number_from_bytes(&key_block_info[i..i + self.number_width])?;
            i += self.number_width;
            
            // key block decompressed size
            let key_block_decompressed_size = self.read_number_from_bytes(&key_block_info[i..i + self.number_width])?;
            i += self.number_width;
            
            key_block_info_list.push((key_block_compressed_size, key_block_decompressed_size));
        }
        
        Ok(key_block_info_list)
    }
    
    /// Decode key block
    /// Port of _decode_key_block from readmdict.py:189-218
    fn decode_key_block(&self, key_block_compressed: &[u8], key_block_info_list: &[(u64, u64)]) -> Result<Vec<(u64, Vec<u8>)>> {
        let mut key_list = Vec::new();
        let mut i = 0;
        
        for &(compressed_size, _decompressed_size) in key_block_info_list {
            let start = i;
            let end = i + compressed_size as usize;
            
            if end > key_block_compressed.len() {
                break;
            }
            
            // 4 bytes: compression type
            let key_block_type = &key_block_compressed[start..start + 4];
            // 4 bytes: adler checksum of decompressed key block
            let adler32_expected = u32::from_be_bytes([
                key_block_compressed[start + 4],
                key_block_compressed[start + 5],
                key_block_compressed[start + 6],
                key_block_compressed[start + 7],
            ]);
            
            let key_block = if key_block_type == b"\x00\x00\x00\x00" {
                // no compression
                key_block_compressed[start + 8..end].to_vec()
            } else if key_block_type == b"\x01\x00\x00\x00" {
                // LZO compression - not supported
                return Err(Error::UnsupportedCompression);
            } else if key_block_type == b"\x02\x00\x00\x00" {
                // zlib compression
                let mut decoder = flate2::read::ZlibDecoder::new(&key_block_compressed[start + 8..end]);
                let mut decompressed = Vec::new();
                std::io::Read::read_to_end(&mut decoder, &mut decompressed)
                    .map_err(|e| Error::Io(e))?;
                decompressed
            } else {
                return Err(Error::Parse("Unknown compression type".to_string()));
            };
            
            // extract one single key block into a key list
            let mut block_keys = self.split_key_block(&key_block)?;
            key_list.append(&mut block_keys);
            
            // verify adler32 checksum
            let mut hasher = Adler32::new();
            hasher.write_slice(&key_block);
            let adler32_actual = hasher.checksum();
            if adler32_expected != adler32_actual {
                return Err(Error::Parse("Key block adler32 checksum mismatch".to_string()));
            }
            
            i += compressed_size as usize;
        }
        
        Ok(key_list)
    }
    
    /// Helper method to read number based on version
    fn read_number_from_bytes(&self, bytes: &[u8]) -> Result<u64> {
        if self.number_width == 4 {
            if bytes.len() < 4 {
                return Err(Error::Parse("Not enough bytes for u32".to_string()));
            }
            Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64)
        } else {
            if bytes.len() < 8 {
                return Err(Error::Parse("Not enough bytes for u64".to_string()));
            }
            Ok(u64::from_be_bytes([
                bytes[0], bytes[1], bytes[2], bytes[3],
                bytes[4], bytes[5], bytes[6], bytes[7],
            ]))
        }
    }
    
    /// Split key block into individual keys
    /// Port of _split_key_block from readmdict.py:219-243
    fn split_key_block(&self, key_block: &[u8]) -> Result<Vec<(u64, Vec<u8>)>> {
        let mut key_list = Vec::new();
        let mut key_start_index = 0;
        
        while key_start_index < key_block.len() {
            if key_start_index + self.number_width > key_block.len() {
                break;
            }
            
            // the corresponding record's offset in record block
            let key_id = self.read_number_from_bytes(&key_block[key_start_index..key_start_index + self.number_width])?;
            
            // key text ends with delimiter
            let (delimiter, width) = if self.encoding == "UTF-16" {
                (b"\x00\x00".as_slice(), 2)
            } else {
                (b"\x00".as_slice(), 1)
            };
            
            let mut i = key_start_index + self.number_width;
            let mut key_end_index = key_block.len();
            
            while i + width <= key_block.len() {
                if &key_block[i..i + width] == delimiter {
                    key_end_index = i;
                    break;
                }
                i += width;
            }
            
            if key_end_index == key_block.len() {
                // No delimiter found, use rest of block
                key_end_index = key_block.len();
            }
            
            let key_bytes = &key_block[key_start_index + self.number_width..key_end_index];
            
            // Decode key text and re-encode as UTF-8
            let key_text = if self.encoding == "UTF-16" {
                let (decoded, _, _) = encoding_rs::UTF_16LE.decode(key_bytes);
                decoded.trim().as_bytes().to_vec()
            } else {
                // For other encodings, try to decode and re-encode as UTF-8
                match self.encoding.as_str() {
                    "UTF-8" => key_bytes.to_vec(),
                    "GB18030" | "GBK" | "GB2312" => {
                        let (decoded, _, _) = encoding_rs::GB18030.decode(key_bytes);
                        decoded.trim().as_bytes().to_vec()
                    }
                    _ => {
                        // Fallback to UTF-8 with replacement
                        let (decoded, _, _) = encoding_rs::UTF_8.decode(key_bytes);
                        decoded.trim().as_bytes().to_vec()
                    }
                }
            };
            
            key_start_index = key_end_index + width;
            key_list.push((key_id, key_text));
        }
        
        Ok(key_list)
    }
}

// MDX struct - equivalent to Python MDX class
// Port of MDX class from readmdict.py:491-599
#[derive(Debug)]
pub struct Mdx {
    mdict: MDict,
    substyle: bool,
}

impl Mdx {
    /// Constructor - direct port of Python MDX.__init__
    /// Port of __init__ from readmdict.py:502-505
    pub fn new(fname: &str, encoding: Option<String>, substyle: bool, passcode: Option<Passcode>) -> Result<Self> {
        let mdict = MDict::new(fname, encoding, passcode)?;
        Ok(Self { mdict, substyle })
    }
    
    /// Items iterator - direct port of Python MDX.items
    /// Port of items from readmdict.py:506-508
    pub fn items(&self) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
        self.decode_record_block()
    }
    
    /// Decode record block - direct port of Python MDX._decode_record_block
    /// Port of _decode_record_block from readmdict.py:523-599
    fn decode_record_block(&self) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
        let mut file = std::fs::File::open(&self.mdict.fname)?;
        file.seek(std::io::SeekFrom::Start(self.mdict.record_block_offset))?;
        
        let num_record_blocks = self.mdict.read_number(&mut file)?;
        let num_entries = self.mdict.read_number(&mut file)?;
        
        // Use actual key count instead of declared count
        let actual_entries = self.mdict.key_list.len() as u64;
        if num_entries != actual_entries {
            // Log warning but continue with actual count like Python version
            eprintln!("Warning: Declared entries ({}) != actual keys ({}) - using actual key count", 
                     num_entries, actual_entries);
        }
        
        let record_block_info_size = self.mdict.read_number(&mut file)?;
        let record_block_size = self.mdict.read_number(&mut file)?;
        
        // record block info section
        let mut record_block_info_list = Vec::new();
        let mut size_counter = 0;
        for _ in 0..num_record_blocks {
            let compressed_size = self.mdict.read_number(&mut file)?;
            let decompressed_size = self.mdict.read_number(&mut file)?;
            record_block_info_list.push((compressed_size, decompressed_size));
            size_counter += self.mdict.number_width * 2;
        }
        if size_counter != record_block_info_size as usize {
            return Err(Error::Parse("Record block info size mismatch".to_string()));
        }
        
        // actual record block data
        let mut result = Vec::new();
        let mut offset = 0u64;
        let mut i = 0;
        let mut size_counter = 0u64;
        
        for (compressed_size, decompressed_size) in record_block_info_list {
            let mut record_block_compressed = vec![0u8; compressed_size as usize];
            std::io::Read::read_exact(&mut file, &mut record_block_compressed)?;
            
            // 4 bytes: compression type
            let record_block_type = &record_block_compressed[0..4];
            // 4 bytes adler32 checksum of uncompressed content
            let adler32_expected = u32::from_be_bytes([
                record_block_compressed[4],
                record_block_compressed[5],
                record_block_compressed[6],
                record_block_compressed[7],
            ]);
            
            let record_block = if record_block_type == b"\x00\x00\x00\x00" {
                // no compression
                record_block_compressed[8..].to_vec()
            } else if record_block_type == b"\x01\x00\x00\x00" {
                // lzo compression - not supported
                return Err(Error::UnsupportedCompression);
            } else if record_block_type == b"\x02\x00\x00\x00" {
                // zlib compression
                let mut decoder = flate2::read::ZlibDecoder::new(&record_block_compressed[8..]);
                let mut decompressed = Vec::new();
                std::io::Read::read_to_end(&mut decoder, &mut decompressed)?;
                decompressed
            } else {
                return Err(Error::Parse("Unknown compression type".to_string()));
            };
            
            // verify adler32 checksum
            let mut hasher = Adler32::new();
            hasher.write_slice(&record_block);
            let adler32_actual = hasher.checksum();
            if adler32_expected != adler32_actual {
                return Err(Error::Parse("Record block adler32 checksum mismatch".to_string()));
            }
            
            if record_block.len() != decompressed_size as usize {
                return Err(Error::Parse("Decompressed size mismatch".to_string()));
            }
            
            // split record block according to the offset info from key block
            while i < self.mdict.key_list.len() {
                let (record_start, key_text) = &self.mdict.key_list[i];
                // reach the end of current record block
                if *record_start >= offset + record_block.len() as u64 {
                    break;
                }
                // record end index
                let record_end = if i < self.mdict.key_list.len() - 1 {
                    self.mdict.key_list[i + 1].0
                } else {
                    record_block.len() as u64 + offset
                };
                i += 1;
                
                let start_idx = (*record_start - offset) as usize;
                let end_idx = (record_end - offset) as usize;
                if start_idx < record_block.len() && end_idx <= record_block.len() {
                    let record = record_block[start_idx..end_idx].to_vec();
                    
                    // convert to utf-8 (for MDX, records are text)
                    let record_text = if self.mdict.encoding == "UTF-16" {
                        let (decoded, _, _) = encoding_rs::UTF_16LE.decode(&record);
                        decoded.trim_end_matches('\0').trim().as_bytes().to_vec()
                    } else {
                        match self.mdict.encoding.as_str() {
                            "UTF-8" => {
                                let text = String::from_utf8_lossy(&record);
                                text.trim_end_matches('\0').trim().as_bytes().to_vec()
                            }
                            "GB18030" | "GBK" | "GB2312" => {
                                let (decoded, _, _) = encoding_rs::GB18030.decode(&record);
                                decoded.trim_end_matches('\0').trim().as_bytes().to_vec()
                            }
                            _ => {
                                let (decoded, _, _) = encoding_rs::UTF_8.decode(&record);
                                decoded.trim_end_matches('\0').trim().as_bytes().to_vec()
                            }
                        }
                    };
                    
                    // substitute styles if enabled
                    let final_record = if self.substyle && !self.mdict.stylesheet.is_empty() {
                        self.substitute_stylesheet(&record_text)?
                    } else {
                        record_text
                    };
                    
                    result.push((key_text.clone(), final_record));
                }
            }
            offset += record_block.len() as u64;
            size_counter += compressed_size;
        }
        
        if size_counter != record_block_size {
            return Err(Error::Parse("Record block size mismatch".to_string()));
        }
        
        Ok(result)
    }
    
    /// Substitute stylesheet - direct port of Python MDX._substitute_stylesheet
    /// Port of _substitute_stylesheet from readmdict.py:510-522
    fn substitute_stylesheet(&self, txt: &[u8]) -> Result<Vec<u8>> {
        // Convert bytes to string for regex processing
        let text = String::from_utf8_lossy(txt);
        
        // Find style tags like `1`, `2`, etc.
        let re_split = regex::Regex::new(r"`\d+`").map_err(|e| Error::Parse(format!("Regex error: {}", e)))?;
        let re_find = regex::Regex::new(r"`\d+`").map_err(|e| Error::Parse(format!("Regex error: {}", e)))?;
        
        let txt_list: Vec<&str> = re_split.split(&text).collect();
        let txt_tags: Vec<&str> = re_find.find_iter(&text).map(|m| m.as_str()).collect();
        
        let mut txt_styled = txt_list[0].to_string();
        
        for (j, p) in txt_list[1..].iter().enumerate() {
            if j < txt_tags.len() {
                let tag_num = &txt_tags[j][1..txt_tags[j].len()-1]; // Remove backticks
                if let Some((style_begin, style_end)) = self.mdict.stylesheet.get(tag_num) {
                    if p.ends_with('\n') {
                        txt_styled.push_str(style_begin);
                        txt_styled.push_str(p.trim_end());
                        txt_styled.push_str(style_end);
                        txt_styled.push_str("\r\n");
                    } else {
                        txt_styled.push_str(style_begin);
                        txt_styled.push_str(p);
                        txt_styled.push_str(style_end);
                    }
                } else {
                    txt_styled.push_str(p);
                }
            } else {
                txt_styled.push_str(p);
            }
        }
        
        Ok(txt_styled.into_bytes())
    }
    
    /// Delegate methods to MDict
    pub fn len(&self) -> usize {
        self.mdict.len()
    }
    
    pub fn is_empty(&self) -> bool {
        self.mdict.is_empty()
    }
    
    pub fn keys(&self) -> impl Iterator<Item = &[u8]> {
        self.mdict.keys()
    }
    
    pub fn header(&self) -> &HashMap<String, String> {
        self.mdict.header()
    }
}

// MDD struct - equivalent to Python MDD class
// Port of MDD class from readmdict.py:405-490
#[derive(Debug)]
pub struct Mdd {
    mdict: MDict,
}

impl Mdd {
    /// Constructor - direct port of Python MDD.__init__
    /// Port of __init__ from readmdict.py:416-417
    pub fn new(fname: &str, passcode: Option<Passcode>) -> Result<Self> {
        let mdict = MDict::new(fname, Some("UTF-16".to_string()), passcode)?;
        Ok(Self { mdict })
    }
    
    /// Items iterator - direct port of Python MDD.items
    /// Port of items from readmdict.py:419-421
    pub fn items(&self) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
        self.decode_record_block()
    }
    
    /// Decode record block - direct port of Python MDD._decode_record_block
    /// Port of _decode_record_block from readmdict.py:423-490
    fn decode_record_block(&self) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
        let mut file = std::fs::File::open(&self.mdict.fname)?;
        file.seek(std::io::SeekFrom::Start(self.mdict.record_block_offset))?;
        
        let num_record_blocks = self.mdict.read_number(&mut file)?;
        let num_entries = self.mdict.read_number(&mut file)?;
        if num_entries != self.mdict.num_entries as u64 {
            return Err(Error::Parse("Number of entries mismatch".to_string()));
        }
        let record_block_info_size = self.mdict.read_number(&mut file)?;
        let record_block_size = self.mdict.read_number(&mut file)?;
        
        // record block info section
        let mut record_block_info_list = Vec::new();
        let mut size_counter = 0;
        for _ in 0..num_record_blocks {
            let compressed_size = self.mdict.read_number(&mut file)?;
            let decompressed_size = self.mdict.read_number(&mut file)?;
            record_block_info_list.push((compressed_size, decompressed_size));
            size_counter += self.mdict.number_width * 2;
        }
        if size_counter != record_block_info_size as usize {
            return Err(Error::Parse("Record block info size mismatch".to_string()));
        }
        
        // actual record block
        let mut result = Vec::new();
        let mut offset = 0u64;
        let mut i = 0;
        let mut size_counter = 0u64;
        
        for (compressed_size, decompressed_size) in record_block_info_list {
            let mut record_block_compressed = vec![0u8; compressed_size as usize];
            std::io::Read::read_exact(&mut file, &mut record_block_compressed)?;
            
            // 4 bytes: compression type
            let record_block_type = &record_block_compressed[0..4];
            // 4 bytes: adler32 checksum of decompressed record block
            let adler32_expected = u32::from_be_bytes([
                record_block_compressed[4],
                record_block_compressed[5],
                record_block_compressed[6],
                record_block_compressed[7],
            ]);
            
            let record_block = if record_block_type == b"\x00\x00\x00\x00" {
                // no compression
                record_block_compressed[8..].to_vec()
            } else if record_block_type == b"\x01\x00\x00\x00" {
                // LZO compression - not supported
                return Err(Error::UnsupportedCompression);
            } else if record_block_type == b"\x02\x00\x00\x00" {
                // zlib compression
                let mut decoder = flate2::read::ZlibDecoder::new(&record_block_compressed[8..]);
                let mut decompressed = Vec::new();
                std::io::Read::read_to_end(&mut decoder, &mut decompressed)?;
                decompressed
            } else {
                return Err(Error::Parse("Unknown compression type".to_string()));
            };
            
            // verify adler32 checksum
            let mut hasher = Adler32::new();
            hasher.write_slice(&record_block);
            let adler32_actual = hasher.checksum();
            if adler32_expected != adler32_actual {
                return Err(Error::Parse("Record block adler32 checksum mismatch".to_string()));
            }
            
            if record_block.len() != decompressed_size as usize {
                return Err(Error::Parse("Decompressed size mismatch".to_string()));
            }
            
            // split record block according to the offset info from key block
            while i < self.mdict.key_list.len() {
                let (record_start, key_text) = &self.mdict.key_list[i];
                // reach the end of current record block
                if *record_start >= offset + record_block.len() as u64 {
                    break;
                }
                // record end index
                let record_end = if i < self.mdict.key_list.len() - 1 {
                    self.mdict.key_list[i + 1].0
                } else {
                    record_block.len() as u64 + offset
                };
                i += 1;
                
                let start_idx = (*record_start - offset) as usize;
                let end_idx = (record_end - offset) as usize;
                if start_idx < record_block.len() && end_idx <= record_block.len() {
                    let data = record_block[start_idx..end_idx].to_vec();
                    result.push((key_text.clone(), data));
                }
            }
            offset += record_block.len() as u64;
            size_counter += compressed_size;
        }
        
        if size_counter != record_block_size {
            return Err(Error::Parse("Record block size mismatch".to_string()));
        }
        
        Ok(result)
    }
    
    /// Delegate methods to MDict
    pub fn len(&self) -> usize {
        self.mdict.len()
    }
    
    pub fn is_empty(&self) -> bool {
        self.mdict.is_empty()
    }
    
    pub fn keys(&self) -> impl Iterator<Item = &[u8]> {
        self.mdict.keys()
    }
    
    pub fn header(&self) -> &HashMap<String, String> {
        self.mdict.header()
    }
}