uqa-analysis 0.3.5

Tokenizers, char/token filters, and analyzers for UQA full-text search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

use super::*;
use crate::kuromoji::{DictionaryArtifact, DictionaryBytes, ResourceHash, ResourceLimits};
use crate::{AnalyzerLimits, AnalyzerResources, NormalizationConfig, Tokenizer};
use parking_lot::Mutex;

mod corpus;
mod resources;
mod tokenizer;

#[test]
fn normalization_profiles_snapshot_alias_changes_and_restore_only_exact_artifacts() {
    let current = Arc::new(Mutex::new(Some(DictionaryBytes::Static(
        uqa_kuromoji_data::BUNDLE,
    ))));
    let requests = Arc::new(Mutex::new(Vec::new()));
    let kuromoji = KuromojiResources::with_resolver(
        Arc::new({
            let current = current.clone();
            let requests = requests.clone();
            move |request: &DictionaryRequest| {
                requests.lock().push(request.clone());
                Ok(current.lock().clone().map(|bytes| DictionaryArtifact {
                    sha256: ResourceHash::of(bytes.as_ref()),
                    bytes,
                }))
            }
        }),
        ResourceLimits {
            max_cached_dictionaries: 0,
            ..Default::default()
        },
    );
    let owner = || {
        AnalyzerResources::builder(AnalyzerLimits::default())
            .kuromoji_resources(kuromoji.clone())
            .build()
    };
    let resources = owner();
    let config = Analyzer::new(
        Tokenizer::Kuromoji(KuromojiTokenizerConfig {
            dictionary: "current".into(),
            ..Default::default()
        }),
        vec![
            crate::TokenFilter::Stop {
                language: String::new(),
                custom_words: Vec::new(),
            },
            crate::TokenFilter::UnicodeSimpleLowercase(crate::SimpleLowercaseConfig {
                unicode_profile: UnicodeProfile::Kuromoji {
                    dictionary: "current".into(),
                }
                .into(),
            }),
        ],
        Vec::new(),
    )
    .with_normalization(NormalizationConfig::UnicodeSimpleLowercase {
        profile: UnicodeProfile::Kuromoji {
            dictionary: "current".into(),
        },
    });
    let retained = resources.compile(&config).unwrap();
    assert_eq!(
        requests.lock().as_slice(),
        [DictionaryRequest::Name("current".into())]
    );
    assert_eq!(retained.normalize("UQA").unwrap(), "uqa");
    assert_eq!(retained.analyze("UQA").unwrap(), ["uqa"]);
    let reopened = owner()
        .restore_json(retained.descriptor().canonical_json())
        .unwrap();
    assert_eq!(
        requests.lock()[1],
        DictionaryRequest::Sha256(uqa_kuromoji_data::BUNDLE_SHA256.parse().unwrap())
    );
    assert_eq!(reopened.normalize("UQA").unwrap(), "uqa");
    let replacement = crate::kuromoji::tests::fixtures::bundle();
    *current.lock() = Some(DictionaryBytes::Shared(replacement.into()));
    let changed = resources.compile(&config).unwrap();
    assert_ne!(
        changed.descriptor().fingerprint(),
        retained.descriptor().fingerprint()
    );
    assert_eq!(changed.normalize("UQA").unwrap(), "UQA");
    assert_eq!(changed.analyze("UQA").unwrap(), ["UQA"]);
    assert_eq!(retained.normalize("UQA").unwrap(), "uqa");
    assert_eq!(retained.analyze("UQA").unwrap(), ["uqa"]);
    let fresh = owner();
    assert!(fresh
        .restore_json(retained.descriptor().canonical_json())
        .is_err());
    assert_eq!(fresh.cache_stats().analyzers, 0);
    *current.lock() = None;
    assert!(resources.compile(&config).is_err());
    assert!(owner()
        .restore_json(changed.descriptor().canonical_json())
        .is_err());
    assert!(Arc::ptr_eq(
        &retained,
        &resources
            .restore_json(retained.descriptor().canonical_json())
            .unwrap()
    ));
    assert_eq!(retained.normalize("UQA").unwrap(), "uqa");
    assert_eq!(changed.normalize("UQA").unwrap(), "UQA");
    assert_eq!(resources.cache_stats().analyzers, 2);
}