finalfrontier 0.9.4

Train/use word embeddings with subword units
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

use std::iter::FusedIterator;
use std::{option, slice};

/// A single lookup index.
#[derive(Copy, Clone)]
pub struct SingleIdx {
    idx: u64,
}

impl SingleIdx {
    pub fn new(idx: u64) -> Self {
        SingleIdx { idx }
    }

    pub fn idx(self) -> u64 {
        self.idx
    }
}

/// A lookup index with associated subword indices.
#[derive(Clone)]
pub struct WordWithSubwordsIdx {
    word_idx: u64,
    subwords: Vec<u64>,
}

impl WordWithSubwordsIdx {
    pub fn new(word_idx: u64, subwords: impl Into<Vec<u64>>) -> Self {
        WordWithSubwordsIdx {
            word_idx,
            subwords: subwords.into(),
        }
    }
}

/// Vocabulary indexing trait.
///
/// This trait defines methods shared by indexing types.
pub trait WordIdx: Clone {
    /// Return the unique word index for the WordIdx.
    fn word_idx(&self) -> u64;

    /// Build a new WordIdx containing only a single index.
    fn from_word_idx(word_idx: u64) -> Self;

    /// Return the number of indices.
    fn len(&self) -> usize;

    /// Return whether this is empty.
    fn is_empty(&self) -> bool {
        self.len() == 0
    }
}

impl WordIdx for SingleIdx {
    fn word_idx(&self) -> u64 {
        self.idx
    }

    fn from_word_idx(word_idx: u64) -> Self {
        SingleIdx::new(word_idx)
    }

    fn len(&self) -> usize {
        1
    }
}

impl<'a> IntoIterator for &'a SingleIdx {
    type Item = u64;
    type IntoIter = option::IntoIter<u64>;

    fn into_iter(self) -> Self::IntoIter {
        Some(self.idx).into_iter()
    }
}

impl WordIdx for WordWithSubwordsIdx {
    fn word_idx(&self) -> u64 {
        self.word_idx
    }

    fn from_word_idx(word_idx: u64) -> Self {
        WordWithSubwordsIdx {
            word_idx,
            subwords: Vec::new(),
        }
    }

    fn len(&self) -> usize {
        1 + self.subwords.len()
    }
}

impl<'a> IntoIterator for &'a WordWithSubwordsIdx {
    type Item = u64;
    type IntoIter = IdxIter<'a>;

    fn into_iter(self) -> Self::IntoIter {
        IdxIter {
            word_idx: Some(self.word_idx),
            subwords: self.subwords.iter(),
        }
    }
}

/// Iterator over Indices.
pub struct IdxIter<'a> {
    word_idx: Option<u64>,
    subwords: slice::Iter<'a, u64>,
}

impl<'a> Iterator for IdxIter<'a> {
    type Item = u64;

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(idx) = self.subwords.next() {
            Some(*idx)
        } else {
            self.word_idx.take()
        }
    }
}

impl<'a> FusedIterator for IdxIter<'a> {}

#[cfg(test)]
mod test {
    use crate::idx::{SingleIdx, WordIdx, WordWithSubwordsIdx};

    #[test]
    fn test_idx_iter() {
        let with_subwords = WordWithSubwordsIdx::new(0, vec![24, 4, 42]);
        let mut idx_iter = (&with_subwords).into_iter();
        assert_eq!(24, idx_iter.next().unwrap());
        assert_eq!(4, idx_iter.next().unwrap());
        assert_eq!(42, idx_iter.next().unwrap());
        assert_eq!(0, idx_iter.next().unwrap());
        assert_eq!(0, with_subwords.word_idx());

        let single = SingleIdx::from_word_idx(0);
        let mut idx_iter = (&single).into_iter();
        assert_eq!(0, idx_iter.next().unwrap());
        assert_eq!(0, single.word_idx());
    }
}