uqa-analysis 0.3.5

Tokenizers, char/token filters, and analyzers for UQA full-text search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Japanese lattice phrases and emitted segment morphology have separate lookup paths.

use crate::kuromoji::{KuromojiDictionary, UserDictionary};
use crate::morphology::lattice::WordId;

use super::KuromojiOrigin;

pub(super) struct Costs {
    pub left: u16,
    pub right: u16,
    pub word: i32,
}

pub(super) fn costs(
    id: WordId,
    model: &KuromojiDictionary,
    user: Option<&UserDictionary>,
) -> Costs {
    match id {
        WordId::Known(id) | WordId::Unknown(id) => {
            let word = model.word(id).expect("validated lattice word");
            Costs {
                left: word.left_context(),
                right: word.right_context(),
                word: i32::from(word.cost()),
            }
        }
        WordId::User(id) => {
            let phrase = user
                .expect("selected user model")
                .entry(id)
                .expect("matched phrase");
            Costs {
                left: phrase.left_context(),
                right: phrase.right_context(),
                word: phrase.cost(),
            }
        }
    }
}

#[derive(Clone, Copy)]
pub(super) enum TokenWord {
    Dictionary(u32, KuromojiOrigin),
    UserSegment(u32),
}

impl TokenWord {
    pub fn origin(self) -> KuromojiOrigin {
        match self {
            Self::Dictionary(_, origin) => origin,
            Self::UserSegment(_) => KuromojiOrigin::User,
        }
    }

    pub fn attributes<'a>(
        self,
        model: &'a KuromojiDictionary,
        user: Option<&'a UserDictionary>,
    ) -> (
        [Option<&'a str>; 6],
        crate::kuromoji::attributes::AttributeErrors,
    ) {
        match self {
            Self::Dictionary(id, _) => {
                let word = model.word(id).expect("validated emitted word");
                (
                    [
                        Some(word.part_of_speech()),
                        word.base_form(),
                        word.reading(),
                        word.pronunciation(),
                        word.inflection_type(),
                        word.inflection_form(),
                    ],
                    crate::kuromoji::attributes::AttributeErrors::default(),
                )
            }
            Self::UserSegment(id) => {
                let word = user
                    .expect("selected user model")
                    .word(id)
                    .expect("validated emitted segment");
                let mut errors = crate::kuromoji::attributes::AttributeErrors::default();
                let pos = word.feature_value(1);
                if pos.is_none() {
                    errors.record(0);
                }
                let reading = word.feature_value(0);
                if reading.is_none() {
                    errors.record(2);
                }
                ([pos, None, reading, None, None, None], errors)
            }
        }
    }
}