Skip to main content

harper_brill/
lib.rs

1use std::num::NonZero;
2use std::rc::Rc;
3use std::sync::{Arc, LazyLock};
4
5pub use harper_pos_utils::{
6    BrillChunker, BrillTagger, BurnChunkerCpu, CachedChunker, Chunker, FreqDict, Tagger, UPOS,
7};
8
9const BRILL_TAGGER_SOURCE: &str = include_str!("../trained_tagger_model.json");
10
11static BRILL_TAGGER: LazyLock<Arc<BrillTagger<FreqDict>>> =
12    LazyLock::new(|| Arc::new(uncached_brill_tagger()));
13
14fn uncached_brill_tagger() -> BrillTagger<FreqDict> {
15    serde_json::from_str(BRILL_TAGGER_SOURCE).unwrap()
16}
17
18/// Get a copy of a shared, lazily-initialized [`BrillTagger`]. There will be only one instance
19/// per-process.
20pub fn brill_tagger() -> Arc<BrillTagger<FreqDict>> {
21    (*BRILL_TAGGER).clone()
22}
23
24const BRILL_CHUNKER_SOURCE: &str = include_str!("../trained_chunker_model.json");
25
26static BRILL_CHUNKER: LazyLock<Arc<BrillChunker>> =
27    LazyLock::new(|| Arc::new(uncached_brill_chunker()));
28
29fn uncached_brill_chunker() -> BrillChunker {
30    serde_json::from_str(BRILL_CHUNKER_SOURCE).unwrap()
31}
32
33/// Get a copy of a shared, lazily-initialized [`BrillChunker`]. There will be only one instance
34/// per-process.
35pub fn brill_chunker() -> Arc<BrillChunker> {
36    (*BRILL_CHUNKER).clone()
37}
38
39const BURN_CHUNKER_VOCAB: &[u8; 627993] = include_bytes!("../finished_chunker/vocab.json");
40const BURN_CHUNKER_BIN: &[u8; 806312] = include_bytes!("../finished_chunker/model.mpk");
41
42thread_local! {
43    static BURN_CHUNKER: Rc<CachedChunker<BurnChunkerCpu>> =  Rc::new(uncached_burn_chunker());
44}
45
46fn uncached_burn_chunker() -> CachedChunker<BurnChunkerCpu> {
47    CachedChunker::new(
48        BurnChunkerCpu::load_from_bytes_cpu(BURN_CHUNKER_BIN, BURN_CHUNKER_VOCAB, 6, 0.3),
49        NonZero::new(10000).unwrap(),
50    )
51}
52
53/// Get a copy of a shared, lazily-initialized [`BurnChunkerCpu`]. There will be only one instance
54/// per-process. Since neural net inference is extremely expensive, this chunker is memoized as
55/// well.
56pub fn burn_chunker() -> Rc<CachedChunker<BurnChunkerCpu>> {
57    (BURN_CHUNKER).with(|c| c.clone())
58}