wordfreq-model 0.2.3

Model loaders for wordfreq-rs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

targets = [
    ('large', 'ar'),
    ('large', 'bn'),
    ('large', 'ca'),
    ('large', 'cs'),
    ('large', 'de'),
    ('large', 'en'),
    ('large', 'es'),
    ('large', 'fi'),
    ('large', 'fr'),
    ('large', 'he'),
    ('large', 'it'),
    ('large', 'ja'),
    ('large', 'mk'),
    ('large', 'nb'),
    ('large', 'nl'),
    ('large', 'pl'),
    ('large', 'pt'),
    ('large', 'ru'),
    ('large', 'sv'),
    ('large', 'uk'),
    ('large', 'zh'),
    ('small', 'ar'),
    ('small', 'bg'),
    ('small', 'bn'),
    ('small', 'ca'),
    ('small', 'cs'),
    ('small', 'da'),
    ('small', 'de'),
    ('small', 'el'),
    ('small', 'en'),
    ('small', 'es'),
    ('small', 'fa'),
    ('small', 'fi'),
    ('small', 'fil'),
    ('small', 'fr'),
    ('small', 'he'),
    ('small', 'hi'),
    ('small', 'hu'),
    ('small', 'id'),
    ('small', 'is'),
    ('small', 'it'),
    ('small', 'ja'),
    ('small', 'ko'),
    ('small', 'lt'),
    ('small', 'lv'),
    ('small', 'mk'),
    ('small', 'ms'),
    ('small', 'nb'),
    ('small', 'nl'),
    ('small', 'pl'),
    ('small', 'pt'),
    ('small', 'ro'),
    ('small', 'ru'),
    ('small', 'sh'),
    ('small', 'sk'),
    ('small', 'sl'),
    ('small', 'sv'),
    ('small', 'ta'),
    ('small', 'tr'),
    ('small', 'uk'),
    ('small', 'ur'),
    ('small', 'vi'),
    ('small', 'zh'),
]

TAB = ' ' * 4

#########################
# Cargo.toml
#########################

wordfreq_version = '0.2'

with open('templates/cargo_toml.txt', 'rt') as f:
    cargo_toml = f.read()

with open('Cargo.toml', 'wt') as f:
    features_block = []
    for wordlist, lang in targets:
        features_block.append(f'{wordlist}-{lang} = []')
    f.write(
        cargo_toml.format(
            features_block='\n'.join(features_block),
            wordfreq_version=wordfreq_version,
        )
    )

#########################
# build.rs
#########################

with open('templates/build_rs.txt', 'rt') as f:
    build_rs = f.read()

with open('build.rs', 'wt') as f:
    main_block = []
    for wordlist, lang in targets:
        main_block.append(f'{TAB}#[cfg(feature = "{wordlist}-{lang}")]')
        main_block.append(f'{TAB}build("{wordlist}_{lang}")?;')
    f.write(build_rs.format(main_block='\n'.join(main_block)))

#########################
# src/lib.rs
#########################

with open('templates/lib_rs.txt', 'rt') as f:
    lib_rs = f.read()

with open('src/lib.rs', 'wt') as f:
    const_block = []
    for wordlist, lang in targets:
        const_block.append(f'#[cfg(feature = "{wordlist}-{lang}")]')
        const_block.append(
            f'const DATA_{wordlist.upper()}_{lang.upper()}: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/{wordlist}_{lang}.bin"));'
        )
    model_kind_block = []
    for wordlist, lang in targets:
        model_kind_block.append(f'{TAB}#[cfg(feature = "{wordlist}-{lang}")]')
        model_kind_block.append(f'{TAB}{wordlist.capitalize()}{lang.capitalize()},')
    match_block = []
    for wordlist, lang in targets:
        match_block.append(f'{TAB}{TAB}#[cfg(feature = "{wordlist}-{lang}")]')
        match_block.append(
            f'{TAB}{TAB}ModelKind::{wordlist.capitalize()}{lang.capitalize()} => {{'
        )
        match_block.append(
            f'{TAB}{TAB}{TAB}Ok(WordFreq::deserialize(DATA_{wordlist.upper()}_{lang.upper()})?.standardizer(Standardizer::new("{lang}")?))'
        )
        match_block.append(f'{TAB}{TAB}}}')
    f.write(
        lib_rs.format(
            model_kind_block='\n'.join(model_kind_block),
            const_block='\n'.join(const_block),
            match_block='\n'.join(match_block),
        )
    )