mdictlib 0.2.8

Library-first Rust parser for MDict .mdx and .mdd dictionaries
Documentation
//! Byte-level decode-and-re-encode conformance.
//!
//! Every re-encodable source shape is written, reopened, re-encoded, and
//! compared byte-for-byte with the original. Uncompressed sources must come
//! back identical; zlib sources may differ inside compressed streams only,
//! so they are gated by idempotence (a second pass reproduces the first)
//! and logical equivalence instead.

mod support;

use std::fs;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};

use mdictlib::{
    MddFile, MdxBuilder, MdxFile, WriteCompression, WriteOptions, reencode_mdd, reencode_mdx,
};
use support::{FixtureBuilder, FixtureCompression, FixtureEncoding};

static NEXT_TEMP: AtomicU64 = AtomicU64::new(0);

struct TempFile(PathBuf);

impl TempFile {
    fn create(extension: &str, bytes: &[u8]) -> Self {
        let path = std::env::temp_dir().join(format!(
            "mdictlib-reencode-{}-{}-{extension}",
            std::process::id(),
            NEXT_TEMP.fetch_add(1, Ordering::SeqCst),
        ));
        fs::write(&path, bytes).unwrap();
        Self(path)
    }

    fn path(&self) -> &PathBuf {
        &self.0
    }
}

impl Drop for TempFile {
    fn drop(&mut self) {
        let _ = fs::remove_file(&self.0);
    }
}

fn reencoded_file_bytes(path: &std::path::Path) -> Vec<u8> {
    if path.extension().is_some_and(|extension| extension == "mdd") {
        let resources = MddFile::open(path).unwrap();
        let mut out = Vec::new();
        reencode_mdd(&resources, &mut out).unwrap();
        out
    } else {
        let dictionary = MdxFile::open(path).unwrap();
        let mut out = Vec::new();
        reencode_mdx(&dictionary, &mut out).unwrap();
        out
    }
}

fn sample_entries() -> Vec<(&'static str, &'static str)> {
    vec![
        ("hello", "<p>an English greeting</p>"),
        ("world", "<p>the earth</p>"),
        ("hello", "<p>a duplicate spelling</p>"),
        ("你好", "<div>中文问候</div>"),
        ("世界", "<div>中文词条</div>"),
        ("a-b", "<i>punctuation key</i>"),
        ("Apple", "<p>capitalized</p>"),
        ("zzz", "last"),
    ]
}

#[test]
fn uncompressed_mdx_reencodes_byte_identically() {
    let entries = sample_entries();
    let total_record_bytes: usize = entries.iter().map(|(_, body)| body.len()).sum();
    let fixture = FixtureBuilder::mdx(entries)
        .compression(FixtureCompression::None)
        .key_blocks([3, 2, 3])
        .record_blocks([total_record_bytes - 40, 40])
        .build();
    let file = fixture.write("reencode-none.mdx");
    let original = fs::read(file.path()).unwrap();
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn uncompressed_utf16le_mdx_reencodes_byte_identically() {
    let fixture = FixtureBuilder::mdx(sample_entries())
        .compression(FixtureCompression::None)
        .encoding(FixtureEncoding::Utf16Le)
        .key_blocks([4, 4])
        .build();
    let file = fixture.write("reencode-utf16.mdx");
    let original = fs::read(file.path()).unwrap();
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn uncompressed_gbk_mdx_reencodes_byte_identically() {
    let entries = [("你好", "问候"), ("世界", "词条"), ("apple", "苹果")];
    let fixture = FixtureBuilder::mdx(entries)
        .compression(FixtureCompression::None)
        .encoding(FixtureEncoding::Gbk)
        .build();
    let file = fixture.write("reencode-gbk.mdx");
    let original = fs::read(file.path()).unwrap();
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn creator_normalized_summaries_survive_reencoding() {
    let fixture = FixtureBuilder::mdx(sample_entries())
        .compression(FixtureCompression::None)
        .key_blocks([4, 4])
        .key_summaries([("hello", "hello"), ("你好", "zzz")])
        .build();
    let file = fixture.write("reencode-summaries.mdx");
    let original = fs::read(file.path()).unwrap();
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn legacy_keyword_index_layout_reencodes_byte_identically() {
    let fixture = FixtureBuilder::mdx(sample_entries())
        .compression(FixtureCompression::None)
        .legacy_keyword_index_layout()
        .build();
    let file = fixture.write("reencode-legacy.mdx");
    let original = fs::read(file.path()).unwrap();
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn empty_mdx_reencodes_byte_identically() {
    let fixture = FixtureBuilder::mdx(Vec::<(String, String)>::new())
        .compression(FixtureCompression::None)
        .build();
    let file = fixture.write("reencode-empty.mdx");
    let original = fs::read(file.path()).unwrap();
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn uncompressed_mdd_reencodes_byte_identically() {
    let entries = [
        ("/style.css".to_owned(), b"body {}".to_vec()),
        ("/logo.png".to_owned(), vec![1, 2, 3, 4]),
        ("/zz/data.bin".to_owned(), vec![0u8; 128]),
    ];
    let total_record_bytes: usize = entries.iter().map(|(_, body)| body.len()).sum();
    let fixture = FixtureBuilder::mdd(entries)
        .compression(FixtureCompression::None)
        .key_blocks([2, 1])
        .record_blocks([total_record_bytes - 8, 8])
        .build();
    let file = fixture.write("reencode-none.mdd");
    let original = fs::read(file.path()).unwrap();
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn builder_written_uncompressed_mdx_reencodes_byte_identically() {
    let options = WriteOptions::new().with_compression(WriteCompression::None);
    let mut builder = MdxBuilder::with_options(options);
    for (key, body) in sample_entries() {
        builder.add_entry(key, body).unwrap();
    }
    let mut original = Vec::new();
    builder.finish(&mut original).unwrap();
    let file = TempFile::create("mdx", &original);
    assert_eq!(reencoded_file_bytes(file.path()), original);
}

#[test]
fn zlib_mdx_reencoding_is_idempotent_and_logically_equal() {
    let fixture = FixtureBuilder::mdx(sample_entries())
        .compression(FixtureCompression::Zlib)
        .key_blocks([3, 5])
        .build();
    let file = fixture.write("reencode-zlib.mdx");
    let original = fs::read(file.path()).unwrap();

    let first = reencoded_file_bytes(file.path());
    let first_file = TempFile::create("mdx", &first);
    let second = reencoded_file_bytes(first_file.path());
    assert_eq!(second, first, "re-encoding is idempotent");

    // The compressed streams may differ from the original; the decoded
    // content must not.
    let source = MdxFile::open(file.path()).unwrap();
    let rebuilt = MdxFile::open(first_file.path()).unwrap();
    assert_eq!(source.len(), rebuilt.len());
    let source_rows: Vec<(String, String)> = source
        .entries()
        .map(|entry| {
            let entry = entry.unwrap();
            (entry.key().to_owned(), entry.text().to_owned())
        })
        .collect();
    let rebuilt_rows: Vec<(String, String)> = rebuilt
        .entries()
        .map(|entry| {
            let entry = entry.unwrap();
            (entry.key().to_owned(), entry.text().to_owned())
        })
        .collect();
    assert_eq!(source_rows, rebuilt_rows);
    if first == original {
        // Some zlib streams re-compress identically; when they do, the
        // whole file must match.
        assert_eq!(second, original);
    }
}

#[test]
fn version_1_sources_are_refused() {
    let fixture = support::v1::V1FixtureBuilder::mdx([("hello", "body")]).build();
    let file = fixture.write("reencode-v1.mdx");
    let dictionary = MdxFile::open(file.path()).unwrap();
    let mut out = Vec::new();
    let error = reencode_mdx(&dictionary, &mut out).unwrap_err();
    assert!(
        error.to_string().contains("version 1"),
        "unexpected error: {error}"
    );
}

#[test]
fn encrypted_keyword_sections_are_refused() {
    let fixture = FixtureBuilder::mdx([("hello", "body")])
        .encrypt_keyword_index()
        .build();
    let file = fixture.write("reencode-encrypted.mdx");
    let dictionary = MdxFile::open(file.path()).unwrap();
    let mut out = Vec::new();
    let error = reencode_mdx(&dictionary, &mut out).unwrap_err();
    assert!(
        error.to_string().contains("encrypted"),
        "unexpected error: {error}"
    );
}

#[cfg(not(feature = "lzo"))]
#[test]
fn lzo_sources_need_the_feature_to_open() {
    let fixture = FixtureBuilder::mdx(sample_entries())
        .compression(FixtureCompression::Lzo)
        .build();
    let file = fixture.write("reencode-lzo.mdx");
    let error = MdxFile::open(file.path()).unwrap_err();
    assert!(
        error.to_string().contains("LZO"),
        "unexpected error: {error}"
    );
}

#[cfg(feature = "lzo")]
#[test]
fn lzo_blocks_are_refused_by_reencode() {
    let fixture = FixtureBuilder::mdx(sample_entries())
        .compression(FixtureCompression::Lzo)
        .build();
    let file = fixture.write("reencode-lzo.mdx");
    let dictionary = MdxFile::open(file.path()).unwrap();
    let mut out = Vec::new();
    let error = reencode_mdx(&dictionary, &mut out).unwrap_err();
    assert!(
        error.to_string().contains("LZO"),
        "unexpected error: {error}"
    );
}