lindera 6.2.0

A morphological analysis library.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

//! The embedded (`include_bytes!`) dictionaries must read their connection
//! cost matrix in place, like the mmap path does.
//!
//! `include_bytes!` guarantees only 1-byte alignment, so `embedded_dictionary!`
//! routes `matrix.mtx` through `include_bytes_aligned!` to raise it. Without
//! that, `ConnectionCostMatrix::load` silently falls back to decoding the
//! payload into an owned `Vec<i16>` -- correct, but it reintroduces exactly
//! the copy #926 removes, and on a `no_std`-free target such as wasm32 (where
//! there is no mmap at all) the embedded path is the only path.

#![cfg(any(
    feature = "embed-ipadic",
    feature = "embed-unidic",
    feature = "embed-sudachidict"
))]

use lindera::dictionary::load_dictionary;

/// Asserts that the embedded dictionary behind `uri` borrows its matrix.
fn assert_embedded_matrix_is_borrowed(uri: &str) {
    let dictionary = load_dictionary(uri).expect("embedded dictionary should load");
    let matrix = &dictionary.connection_cost_matrix;

    assert!(
        matrix.is_zero_copy(),
        "{uri}: the embedded connection cost matrix must be borrowed, not copied. \
         The payload starts at byte 6 of matrix.mtx, so the backing static needs \
         at least 2-byte alignment -- check that embedded_dictionary! still uses \
         include_bytes_aligned! for CONNECTION_DATA."
    );

    // Sanity-check that the borrowed view really is the matrix: a MeCab
    // dictionary always has a defined cost between the BOS/EOS context ids.
    assert_eq!(
        matrix.costs().len() as u32,
        matrix.forward_size * matrix.backward_size
    );
}

#[test]
#[cfg(feature = "embed-ipadic")]
fn embedded_ipadic_matrix_is_zero_copy() {
    assert_embedded_matrix_is_borrowed("embedded://ipadic");
}

#[test]
#[cfg(feature = "embed-unidic")]
fn embedded_unidic_matrix_is_zero_copy() {
    assert_embedded_matrix_is_borrowed("embedded://unidic");
}

#[test]
#[cfg(feature = "embed-sudachidict")]
fn embedded_sudachidict_matrix_is_zero_copy() {
    assert_embedded_matrix_is_borrowed("embedded://sudachidict");
}