wordmarkov 0.1.4

A simple but flexible Markov chain library, specifically for text sentences, which handles punctuation and whitespace
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

/*!
 * Code for the synthesis and representation of sentence tokens.
 *
 * Words, spacing and punctuation are all fundamental parts of a sentence.
 */

/**
 * A token – can be either a Word or a Punct, or the beginning or end of a
 * parsed sentence.
 */
#[derive(Eq, PartialEq, Debug, Clone)]
pub enum Token<'a> {
    /// A word, a sequence of characters bounded by non-words.
    ///
    /// A non-word in this case referring to either [Token::Punct],
    /// [Token::Begin] or [Token::End].
    Word(&'a str),

    /// Punctuation and whitespace.
    Punct(&'a str),

    /// The beginning of a sentence.
    Begin,

    /// The end of a sentence.
    End,
}

impl<'a> Token<'a> {
    /// Recompose a slice or list of Tokens into a String.
    pub fn recompose(tokens: &'a [Token<'a>]) -> String {
        tokens
            .iter()
            .map(|x: &'a Token| -> String {
                let s: &'a str = x.into();
                s.into()
            })
            .collect()
    }
}

impl<'a> From<&Token<'a>> for &'a str {
    /// Converts this token into the equivalent string.
    fn from(tok: &Token<'a>) -> Self {
        match tok {
            Token::Word(s) => s,
            Token::Punct(s) => s,
            Token::Begin => "",
            Token::End => "",
        }
    }
}