static_lang_word_lists/
word_lists.rs

1use std::{
2    borrow::Cow,
3    fs, io,
4    ops::{Deref, Index},
5    path::{Path, PathBuf},
6    slice,
7    sync::LazyLock,
8};
9
10use thiserror::Error;
11
12use crate::{metadata::WordListMetadata, newline_delimited_words};
13
14// TODO: this can be Box<str>
15pub(crate) type Word = String;
16pub(crate) type WordSource = Box<[Word]>;
17
18/// A list of words, with optional additional metadata.
19#[derive(Debug)]
20pub struct WordList {
21    words: EagerOrLazy<WordSource>,
22    /// Metadata associated with this word list.
23    ///
24    /// Includes the word list's name, script (if known), and language (if
25    /// known).
26    ///
27    /// You usually only need to access this directly if you plan to edit the
28    /// metadata, as otherwise you can access metadata from the `WordList`
29    /// directly:
30    /// - [`WordList::name`]
31    /// - [`WordList::script`]
32    /// - [`WordList::language`]
33    pub metadata: WordListMetadata,
34}
35
36impl WordList {
37    /// Load a word list from a file.
38    ///
39    /// The file is expected to contain one word per line, and is accompanied by
40    /// a metadata TOML file.
41    /// A fully specified metadata file may look like this:
42    /// ```toml
43    #[doc = include_str!("../data/aosp/en_Latn.toml")]
44    /// ```
45    /// 
46    /// For more details on the TOML format, see [the GitHub README](https://github.com/googlefonts/fontheight/tree/main/static-lang-word-lists#word-list-metadata-schema).
47    #[allow(clippy::result_large_err)]
48    pub fn load(
49        path: impl AsRef<Path>,
50        metadata_path: impl AsRef<Path>,
51    ) -> Result<Self, WordListError> {
52        let mut word_list = WordList::load_without_metadata(path)?;
53        word_list.metadata = WordListMetadata::load(metadata_path)?;
54        Ok(word_list)
55    }
56
57    /// Load a word list from a file.
58    ///
59    /// The file is expected to contain one word per line.
60    /// Always prefer [`WordList::load`] if metadata is available.
61    #[allow(clippy::result_large_err)]
62    pub fn load_without_metadata(
63        path: impl AsRef<Path>,
64    ) -> Result<Self, WordListError> {
65        let path = path.as_ref();
66        let file_content = fs::read_to_string(path).map_err(|io_err| {
67            WordListError::FailedToRead(path.to_owned(), io_err)
68        })?;
69        let name = path
70            .file_stem()
71            .ok_or_else(|| {
72                WordListError::FailedToRead(
73                    path.to_owned(),
74                    io::Error::new(
75                        io::ErrorKind::InvalidData,
76                        "file name is empty",
77                    ),
78                )
79            })?
80            .to_string_lossy()
81            .replace("/", "_");
82
83        Ok(WordList {
84            metadata: WordListMetadata::new_from_name(name),
85            words: newline_delimited_words(file_content).into(),
86        })
87    }
88
89    /// Create a new word list from an iterable.
90    ///
91    /// Types that `impl Into<WordListMetadata>`:
92    /// - [`&str`] (used as name of word list)
93    /// - [`String`] (used as name of word list)
94    /// - [`WordListMetadata`]
95    #[must_use]
96    pub fn define(
97        name_or_metadata: impl Into<WordListMetadata>,
98        words: impl IntoIterator<Item = impl Into<String>>,
99    ) -> Self {
100        WordList {
101            metadata: name_or_metadata.into(),
102            words: words.into_iter().map(Into::into).collect::<Vec<_>>().into(),
103        }
104    }
105
106    // Used by wordlist! {}
107    #[must_use]
108    pub(crate) const fn new_lazy(
109        metadata: WordListMetadata,
110        words: LazyLock<WordSource>,
111    ) -> Self {
112        WordList {
113            words: EagerOrLazy::Lazy(words),
114            metadata,
115        }
116    }
117
118    // Used by wordlist! when building on docs.rs
119    #[allow(dead_code)]
120    pub(crate) const fn stub() -> Self {
121        WordList {
122            metadata: WordListMetadata {
123                name: Cow::Borrowed("stub"),
124                script: None,
125                language: None,
126            },
127            words: EagerOrLazy::Lazy(LazyLock::new(|| unreachable!())),
128        }
129    }
130
131    /// Get the name of the word list.
132    #[inline]
133    #[must_use]
134    pub fn name(&self) -> &str {
135        self.metadata.name()
136    }
137
138    /// Get the script of the word list, if known.
139    ///
140    /// The script is expected to be an [ISO 15924](https://en.wikipedia.org/wiki/ISO_15924)
141    /// four-letter capitalised code, but this is only guaranteed for built-in
142    /// word lists.
143    #[inline]
144    #[must_use]
145    pub fn script(&self) -> Option<&str> {
146        self.metadata.script()
147    }
148
149    /// Get the language of the word list, if known.
150    ///
151    /// The language is expected to be an [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1)
152    /// two-letter code, but this is only guaranteed for built-in word lists.
153    #[inline]
154    #[must_use]
155    pub fn language(&self) -> Option<&str> {
156        self.metadata.language()
157    }
158
159    /// Iterate through the word list.
160    #[must_use]
161    pub fn iter(&self) -> WordListIter<'_> {
162        WordListIter(self.words.iter())
163    }
164
165    /// Get how many words there are in the word list.
166    #[inline]
167    #[must_use]
168    pub fn len(&self) -> usize {
169        self.words.len()
170    }
171
172    /// Returns `true` if there are no words in the word list.
173    #[inline]
174    #[must_use]
175    pub fn is_empty(&self) -> bool {
176        self.words.is_empty()
177    }
178
179    /// Create a new word list by removing words from an existing one, according
180    /// to the `predicate`.
181    ///
182    /// You can think of this similar to calling [`Vec::retain`], except it
183    /// returns a new list instead of modifying the old one in-place.
184    /// Metadata isn't modified.
185    pub fn filter<F>(&self, mut predicate: F) -> Self
186    where
187        F: FnMut(&str) -> bool,
188    {
189        let reduced_words = self
190            .words
191            .iter()
192            .filter(|word| predicate(word))
193            .cloned()
194            .collect::<Vec<_>>();
195        let reduced_words =
196            EagerOrLazy::Eager(reduced_words.into_boxed_slice());
197        Self {
198            metadata: self.metadata.clone(),
199            words: reduced_words,
200        }
201    }
202}
203
204impl Clone for WordList {
205    /// Returns a duplicate of the value.
206    ///
207    /// Note: this will load the word list for `&self` and the newly returned
208    /// word list.
209    fn clone(&self) -> Self {
210        Self {
211            metadata: self.metadata.clone(),
212            words: EagerOrLazy::Eager(self.words.deref().clone()),
213        }
214    }
215}
216
217impl Index<usize> for WordList {
218    type Output = str;
219
220    fn index(&self, index: usize) -> &Self::Output {
221        self.words.index(index).deref()
222    }
223}
224
225#[derive(Debug)]
226enum EagerOrLazy<T> {
227    Eager(T),
228    Lazy(LazyLock<T>),
229}
230
231impl<T> From<T> for EagerOrLazy<T> {
232    fn from(value: T) -> Self {
233        EagerOrLazy::Eager(value)
234    }
235}
236
237impl<T> Deref for EagerOrLazy<T> {
238    type Target = T;
239
240    fn deref(&self) -> &Self::Target {
241        match self {
242            EagerOrLazy::Eager(e) => e,
243            EagerOrLazy::Lazy(l) => l,
244        }
245    }
246}
247
248impl From<Vec<String>> for EagerOrLazy<WordSource> {
249    fn from(value: Vec<String>) -> Self {
250        Self::Eager(value.into_boxed_slice())
251    }
252}
253
254/// An iterator over a [`WordList`].
255///
256/// Returned by [`WordList::iter`].
257#[derive(Debug)]
258pub struct WordListIter<'a>(slice::Iter<'a, String>);
259
260impl<'a> Iterator for WordListIter<'a> {
261    type Item = &'a str;
262
263    fn next(&mut self) -> Option<Self::Item> {
264        self.0.next().map(String::as_ref)
265    }
266}
267
268impl ExactSizeIterator for WordListIter<'_> {
269    fn len(&self) -> usize {
270        self.0.len()
271    }
272}
273
274impl DoubleEndedIterator for WordListIter<'_> {
275    fn next_back(&mut self) -> Option<Self::Item> {
276        self.0.next_back().map(String::as_ref)
277    }
278}
279
280/// An error encountered while loading a [`WordList`] and its metadata.
281#[derive(Debug, Error)]
282pub enum WordListError {
283    /// Unable to read either the word list or the metadata file.
284    #[error("failed to read from {}: {}", .0.display(), .1)]
285    FailedToRead(PathBuf, io::Error),
286    /// Unable to parse the metadata.
287    #[error("failed to parse metadata from {}: {}", .0.display(), .1)]
288    MetadataError(PathBuf, toml::de::Error),
289}
290
291#[cfg(feature = "rayon")]
292pub(crate) mod rayon {
293    use rayon::iter::{
294        IndexedParallelIterator, ParallelIterator,
295        plumbing::{
296            Consumer, Producer, ProducerCallback, UnindexedConsumer, bridge,
297        },
298    };
299
300    use super::{WordList, WordListIter};
301
302    /// A [`rayon`]-powered parallel iterator over a [`WordList`].
303    ///
304    /// Returned by [`WordList::par_iter`].
305    #[derive(Debug)]
306    pub struct ParWordListIter<'a>(&'a [String]);
307
308    impl<'a> ParallelIterator for ParWordListIter<'a> {
309        type Item = &'a str;
310
311        fn drive_unindexed<C>(self, consumer: C) -> C::Result
312        where
313            C: UnindexedConsumer<Self::Item>,
314        {
315            bridge(self, consumer)
316        }
317
318        fn opt_len(&self) -> Option<usize> {
319            Some(self.0.len())
320        }
321    }
322
323    impl<'a> Producer for ParWordListIter<'a> {
324        type IntoIter = WordListIter<'a>;
325        type Item = &'a str;
326
327        fn into_iter(self) -> Self::IntoIter {
328            WordListIter(self.0.iter())
329        }
330
331        fn split_at(self, index: usize) -> (Self, Self) {
332            let (left, right) = self.0.split_at(index);
333            (ParWordListIter(left), ParWordListIter(right))
334        }
335    }
336
337    impl IndexedParallelIterator for ParWordListIter<'_> {
338        fn len(&self) -> usize {
339            self.0.len()
340        }
341
342        fn drive<C: Consumer<Self::Item>>(self, consumer: C) -> C::Result {
343            bridge(self, consumer)
344        }
345
346        fn with_producer<CB>(self, callback: CB) -> CB::Output
347        where
348            CB: ProducerCallback<Self::Item>,
349        {
350            callback.callback(self)
351        }
352    }
353
354    impl<'a> rayon::iter::IntoParallelIterator for &'a WordList {
355        type Item = &'a str;
356        type Iter = ParWordListIter<'a>;
357
358        fn into_par_iter(self) -> Self::Iter {
359            ParWordListIter(&self.words)
360        }
361    }
362
363    impl WordList {
364        /// Iterate through the word list in parallel with `rayon`.
365        #[must_use]
366        pub fn par_iter(&self) -> ParWordListIter<'_> {
367            ParWordListIter(&self.words)
368        }
369    }
370}
static_lang_word_lists/word_lists.rs

static_lang_word_lists/
word_lists.rs
