mdictlib 0.2.6

Library-first Rust parser for MDict .mdx and .mdd dictionaries
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
use std::fmt;
use std::io::{Seek, Write};
use std::iter::FusedIterator;
use std::ops::ControlFlow;
use std::path::Path;

use crate::core::{MdictFile, RecordDescriptor, RecordIter};
use crate::error::Result;
use crate::format::TextEncoding;
use crate::index::{KeyIndex, KeyIndexBuild, KeyIndexOptions, KeyIndexSourceIdentity};
use crate::lookup::{KeyMatchPage, KeyMatches};
use crate::types::{ContainerKind, Header, KeyEntry, KeyOrdinal, MemoryUsage, OpenOptions};

/// An opened MDX text dictionary.
pub struct MdxFile {
    inner: MdictFile,
}

impl fmt::Debug for MdxFile {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter
            .debug_struct("MdxFile")
            .field("len", &self.len())
            .field("title", &self.header().title())
            .finish_non_exhaustive()
    }
}

/// One decoded MDX entry with its physical identity.
#[derive(PartialEq, Eq)]
pub struct MdxEntry {
    key: KeyEntry,
    text: String,
}

impl fmt::Debug for MdxEntry {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter
            .debug_struct("MdxEntry")
            .field("ordinal", &self.ordinal())
            .field("key", &self.key())
            .field("text_len", &self.text.len())
            .finish()
    }
}

impl MdxEntry {
    fn new(key: KeyEntry, text: String) -> Self {
        Self { key, text }
    }

    /// Returns the entry's physical ordinal.
    pub const fn ordinal(&self) -> KeyOrdinal {
        self.key.ordinal()
    }

    /// Returns the original decoded key.
    pub fn key(&self) -> &str {
        self.key.key()
    }

    /// Returns the physical key row.
    pub fn key_entry(&self) -> &KeyEntry {
        &self.key
    }

    /// Returns the decoded MDX record text.
    pub fn text(&self) -> &str {
        &self.text
    }

    /// Consumes the entry into its key row and decoded text.
    pub fn into_parts(self) -> (KeyEntry, String) {
        (self.key, self.text)
    }
}

impl MdxFile {
    /// Opens an MDX file with default options.
    ///
    /// The path may also be a `file://` URL, which is what a mobile file
    /// picker answers with — iOS hands back `NSURL`s and Android's Storage
    /// Access Framework does the same — including percent-escaped names. Only
    /// local URLs are accepted; one naming another host is refused.
    ///
    /// # Errors
    ///
    /// Returns an error if the file cannot be read, requests an unsupported
    /// format path, exceeds a safety limit, or contains malformed metadata or
    /// indexes, or if a `file://` URL is remote or malformed.
    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
        Self::open_with_options(path, &OpenOptions::new())
    }

    /// Opens an MDX file with reusable explicit options.
    ///
    /// # Errors
    ///
    /// Returns an error under the same conditions as [`Self::open`], including
    /// when an encrypted header cannot be opened with the supplied options.
    pub fn open_with_options(path: impl AsRef<Path>, options: &OpenOptions) -> Result<Self> {
        Ok(Self {
            inner: MdictFile::open(path, ContainerKind::Mdx, options)?,
        })
    }

    /// Returns parsed top-level header metadata.
    pub fn header(&self) -> &Header {
        self.inner.header()
    }

    /// Returns the number of physical dictionary entries.
    pub fn len(&self) -> u64 {
        self.inner.len()
    }

    /// Returns whether the dictionary declares no entries.
    pub fn is_empty(&self) -> bool {
        self.inner.is_empty()
    }

    /// Reports parser memory charged to this open handle's safety budget.
    ///
    /// # Errors
    ///
    /// Returns an error if an internal cache mutex was poisoned by a panic in
    /// another thread.
    pub fn memory_usage(&self) -> Result<MemoryUsage> {
        self.inner.memory_usage()
    }

    /// Iterates over original keys in physical file order without reading
    /// record payloads.
    ///
    /// Block read and decode failures are yielded as an item; after yielding
    /// an error, the iterator is exhausted.
    pub fn keys(&self) -> impl FusedIterator<Item = Result<KeyEntry>> + '_ {
        self.inner.keys()
    }

    /// Returns one physical key row by ordinal without reading its record.
    ///
    /// # Errors
    ///
    /// Returns an error if the corresponding key block cannot be read or is
    /// malformed.
    pub fn key_at(&self, ordinal: KeyOrdinal) -> Result<Option<KeyEntry>> {
        self.inner.key_at_ordinal(ordinal)
    }

    /// Resolves physical key rows for the supplied ordinals in input order.
    /// Repeated and out-of-range ordinals are preserved as repeated values and
    /// `None`, respectively.
    ///
    /// # Errors
    ///
    /// Returns an error if any requested key block cannot be read or is
    /// malformed.
    pub fn keys_at(&self, ordinals: &[KeyOrdinal]) -> Result<Vec<Option<KeyEntry>>> {
        self.inner.keys_at_ordinals(ordinals)
    }

    /// Reads and decodes one MDX entry by physical ordinal.
    ///
    /// # Errors
    ///
    /// Returns an error if its key or record blocks cannot be read, are
    /// malformed, exceed a materialization limit, or contain invalid text.
    pub fn entry_at(&self, ordinal: KeyOrdinal) -> Result<Option<MdxEntry>> {
        self.inner
            .record_at_ordinal(ordinal)?
            .map(|descriptor| self.read_entry(descriptor))
            .transpose()
    }

    /// Locates every physical entry matching a query without reading records.
    ///
    /// Raw-exact matches across the entire file always win. Header-normalized
    /// fallback is considered only when no raw key matches, and duplicates are
    /// returned in ascending physical order.
    ///
    /// # Errors
    ///
    /// Returns an error if the global locator cannot be built because key data
    /// is malformed, unreadable, or exceeds a safety limit.
    pub fn locate(&self, query: &str) -> Result<Option<KeyMatches>> {
        self.inner
            .locate_keys(query)
            .map(|matches| matches.map(KeyMatches::from_located))
    }

    /// Locates a bounded window of matching physical entries without reading
    /// records or materializing the complete duplicate set.
    ///
    /// `offset` and `limit` address the same ascending physical order returned
    /// by [`Self::locate`]. Raw-exact precedence is decided across the complete
    /// normalized equal range before the page is returned. Consequently a
    /// page beyond the end is `Some` but empty and still reports the complete
    /// match count and basis.
    ///
    /// # Errors
    ///
    /// Returns an error if the global locator cannot be built, source key data
    /// is malformed or unreadable, or the requested page exceeds a configured
    /// safety or aggregate-memory limit.
    pub fn locate_page(
        &self,
        query: &str,
        offset: usize,
        limit: usize,
    ) -> Result<Option<KeyMatchPage>> {
        self.inner
            .locate_key_page(query, offset, limit)
            .map(|page| page.map(KeyMatchPage::from_located))
    }

    /// Locates physical entries whose key starts with `prefix` under the
    /// header's own normalization, in normalized order, stopping after `limit`.
    ///
    /// Duplicates are preserved, so a caller presenting completions to a person
    /// should ask for more than it intends to show and collapse repeats itself.
    ///
    /// # Errors
    ///
    /// Returns an error if the global locator cannot be built because key data
    /// is malformed, unreadable, or exceeds a safety limit.
    pub fn prefix_keys(&self, prefix: &str, limit: usize) -> Result<Vec<KeyEntry>> {
        let ordinals = self.inner.locate_prefix(prefix, limit)?;
        Ok(self
            .inner
            .keys_at_ordinals(&ordinals)?
            .into_iter()
            .flatten()
            .collect())
    }

    /// Visits every entry's normalized key in physical order, without copying
    /// it, until the callback breaks.
    ///
    /// This exists so a caller can apply its own search policy — completion
    /// ranking, edit distance, transliteration — across the whole key space
    /// without building and retaining a second copy of it. Resolve a chosen
    /// [`KeyOrdinal`] back to its original key with [`Self::key_at`].
    ///
    /// # Errors
    ///
    /// Returns an error if the global locator cannot be built because key data
    /// is malformed, unreadable, or exceeds a safety limit.
    pub fn scan_normalized_keys<F>(&self, visit: F) -> Result<()>
    where
        F: FnMut(KeyOrdinal, &str) -> ControlFlow<()>,
    {
        self.inner.scan_normalized_keys(visit)
    }

    /// Reads the lightweight source identity used to bind a persistent key
    /// index.
    ///
    /// This reads metadata from the parser-owned file handle without scanning
    /// source contents. The modification time retains the filesystem's native
    /// precision and is represented as signed Unix nanoseconds.
    ///
    /// # Errors
    ///
    /// Returns an error if file length or modification-time metadata cannot be
    /// read, or if the current length differs from the parsed source length.
    pub fn key_index_source_identity(&self) -> Result<KeyIndexSourceIdentity> {
        crate::core::persistent::source_identity(&self.inner)
    }

    /// Builds a complete persistent key index into a caller-owned sink with
    /// bounded sort memory and cancellation checkpoints.
    ///
    /// Does not initialize the global locator used by [`Self::locate`]. The
    /// destination is never read back and must be discarded on any error.
    ///
    /// # Errors
    ///
    /// Returns an error if source data is malformed or changes, a configured
    /// bound is exceeded, scratch or destination I/O fails, or cancellation is
    /// requested.
    pub fn build_key_index<W, C>(
        &self,
        destination: &mut W,
        options: &KeyIndexOptions,
        cancelled: C,
    ) -> Result<KeyIndexBuild>
    where
        W: Write + Seek + ?Sized,
        C: FnMut() -> bool,
    {
        crate::core::persistent::build_to_writer(&self.inner, destination, options, cancelled)
    }

    /// Builds a complete persistent key index at a caller-selected new path.
    ///
    /// The destination is opened with create-new semantics. After streaming the
    /// generated sections, this method flushes and syncs it without rereading it.
    /// Publication or atomic rename of that path remains the application's
    /// responsibility; the create-new file must be discarded whenever this
    /// method returns an error.
    ///
    /// # Errors
    ///
    /// Returns an error under the same conditions as [`Self::build_key_index`],
    /// or when the destination already exists or cannot be synced.
    pub fn build_key_index_to_path<P, C>(
        &self,
        path: P,
        options: &KeyIndexOptions,
        cancelled: C,
    ) -> Result<KeyIndexBuild>
    where
        P: AsRef<Path>,
        C: FnMut() -> bool,
    {
        crate::core::persistent::build_to_path(&self.inner, path, options, cancelled)
    }

    /// Opens a caller-selected persistent key index bound to `expected` and
    /// this dictionary's current source metadata.
    ///
    /// Validates the fixed header at open; section data is checked lazily when
    /// [`KeyIndexOptions::checksum_policy`](crate::KeyIndexOptions::checksum_policy)
    /// is [`ChecksumPolicy::Verify`](crate::ChecksumPolicy::Verify).
    ///
    /// # Errors
    ///
    /// Returns a structured [`crate::KeyIndexRejection`] through
    /// [`crate::Error::KeyIndexRejected`] for an incompatible, stale, corrupt,
    /// or malformed artifact, and an I/O error when it cannot be read.
    pub fn open_key_index(
        &self,
        path: impl AsRef<Path>,
        expected: &KeyIndexSourceIdentity,
        options: &KeyIndexOptions,
    ) -> Result<KeyIndex> {
        crate::core::persistent::open(&self.inner, path, expected, options).map(KeyIndex::new)
    }

    /// Locates every matching physical row through a persistent key index.
    ///
    /// Raw-exact matches win over header-normalized matches, raw digests are
    /// only filters, and every positive row is proved against source key data.
    /// Duplicates retain ascending physical order. Materialized match ordinals
    /// are bounded by [`crate::Limits::locator_bytes`] and remain charged to
    /// the dictionary's aggregate memory budget while the result is alive.
    ///
    /// # Errors
    ///
    /// Returns an error if the index or corresponding source key data fails
    /// validation or cannot be read, or the complete match set exceeds its
    /// configured byte or aggregate-memory ceiling.
    pub fn locate_with_key_index(
        &self,
        index: &KeyIndex,
        query: &str,
    ) -> Result<Option<KeyMatches>> {
        crate::core::persistent::locate(&self.inner, &index.inner, query)
            .map(|matches| matches.map(KeyMatches::from_located))
    }

    /// Locates a bounded window of matching physical rows through a persistent
    /// key index without allocating the complete duplicate set.
    ///
    /// `offset`, ordering, global raw-exact precedence, duplicate identity,
    /// and total-count semantics are identical to [`Self::locate_page`]. Every
    /// row used to decide or populate the result is proved against current
    /// source key data.
    ///
    /// # Errors
    ///
    /// Returns an error if the index or corresponding source key data fails
    /// validation or cannot be read, or the requested page exceeds a
    /// configured safety or aggregate-memory limit.
    pub fn locate_page_with_key_index(
        &self,
        index: &KeyIndex,
        query: &str,
        offset: usize,
        limit: usize,
    ) -> Result<Option<KeyMatchPage>> {
        crate::core::persistent::locate_page(&self.inner, &index.inner, query, offset, limit)
            .map(|page| page.map(KeyMatchPage::from_located))
    }

    /// Locates normalized-prefix rows through a persistent key index.
    ///
    /// Results use normalized order with physical ordinal as the duplicate
    /// tie-break, and each returned row is proved against source key data.
    ///
    /// # Errors
    ///
    /// Returns an error if the index or corresponding source key data fails
    /// validation or cannot be read.
    pub fn prefix_keys_with_index(
        &self,
        index: &KeyIndex,
        prefix: &str,
        limit: usize,
    ) -> Result<Vec<KeyEntry>> {
        crate::core::persistent::prefix(&self.inner, &index.inner, prefix, limit)
    }

    /// Visits normalized keys in physical source order through a persistent
    /// index until `visit` breaks.
    ///
    /// # Errors
    ///
    /// Returns an error if the index fails validation or cannot be read.
    pub fn scan_normalized_keys_with_index<F>(&self, index: &KeyIndex, visit: F) -> Result<()>
    where
        F: FnMut(KeyOrdinal, &str) -> ControlFlow<()>,
    {
        crate::core::persistent::scan(&self.inner, &index.inner, visit)
    }

    /// Looks up and decodes one entry using the dictionary's current key
    /// matching behavior.
    ///
    /// # Errors
    ///
    /// Returns an error if lookup encounters malformed key data or the matched
    /// record cannot be read and decoded.
    pub fn lookup(&self, query: &str) -> Result<Option<MdxEntry>> {
        let Some(matches) = self.locate(query)? else {
            return Ok(None);
        };
        self.entry_at(matches.first())
    }

    /// Iterates lazily over decoded entries in physical file order.
    ///
    /// Block read and decode failures are yielded as an item; after yielding
    /// an error, the iterator is exhausted.
    pub fn entries(&self) -> impl FusedIterator<Item = Result<MdxEntry>> + '_ {
        MdxEntryIter {
            file: self,
            records: self.inner.records(),
            done: false,
        }
    }

    fn read_entry(&self, descriptor: RecordDescriptor) -> Result<MdxEntry> {
        let encoded_len = descriptor
            .end
            .checked_sub(descriptor.start)
            .ok_or(crate::Error::InvalidFormat("record range is inverted"))?;
        let _memory = self.inner.reserve_decoded_record_text(encoded_len)?;
        let bytes = self
            .inner
            .read_record_span(descriptor.start, descriptor.end)?;
        let text = decode_record_text(self.inner.record_text_encoding()?, &bytes)?;
        Ok(MdxEntry::new(descriptor.key, text))
    }
}

struct MdxEntryIter<'a> {
    file: &'a MdxFile,
    records: RecordIter<'a>,
    done: bool,
}

impl Iterator for MdxEntryIter<'_> {
    type Item = Result<MdxEntry>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.done {
            return None;
        }
        let result = self
            .records
            .next()
            .map(|result| result.and_then(|descriptor| self.file.read_entry(descriptor)));
        if result.as_ref().is_some_and(Result::is_err) {
            self.done = true;
        }
        result
    }
}

impl FusedIterator for MdxEntryIter<'_> {}

fn decode_record_text(encoding: TextEncoding, bytes: &[u8]) -> Result<String> {
    let mut text = encoding.decode(bytes, "mdx record")?;
    let trimmed_len = text.trim_end_matches('\0').len();
    text.truncate(trimmed_len);
    Ok(text)
}