wordmarkov 0.1.4

A simple but flexible Markov chain library, specifically for text sentences, which handles punctuation and whitespace
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

//! Useful token and token sentence related code for the Markov chain.

use crate::sentence::token::Token as LexedToken;
use std::collections::LinkedList;
use std::fmt::Display;
use std::fmt::Formatter;
use std::rc::Rc;

/// A Markov token.
#[derive(Eq, PartialEq, Debug)]
pub enum MarkovToken<'a> {
    Begin,
    End,
    Textlet(&'a str),
}

impl<'a> MarkovToken<'a> {
    fn string_ref(&'a self) -> &'a str {
        match self {
            MarkovToken::Textlet(s) => s,
            MarkovToken::Begin => "",
            MarkovToken::End => "",
        }
    }

    pub fn is_empty(&self) -> bool {
        match self {
            MarkovToken::Textlet(s) => s.is_empty(),
            MarkovToken::Begin => true,
            MarkovToken::End => true,
        }
    }

    pub fn len(&self) -> usize {
        match self {
            MarkovToken::Textlet(s) => s.len(),
            MarkovToken::Begin => 0,
            MarkovToken::End => 0,
        }
    }
}

impl<'a> From<&LexedToken<'a>> for MarkovToken<'a> {
    fn from(lexed: &LexedToken<'a>) -> Self {
        match lexed {
            LexedToken::Begin => MarkovToken::Begin,
            LexedToken::End => MarkovToken::End,

            LexedToken::Punct(w) => MarkovToken::Textlet(w),
            LexedToken::Word(w) => MarkovToken::Textlet(w),
        }
    }
}

/// A Markov token, but owned. Only used from MarkovChain.
#[derive(Eq, PartialEq, Debug)]
pub enum MarkovTokenOwned {
    Begin,
    End,
    Textlet(Rc<str>),
}

impl<'a> From<&'a MarkovTokenOwned> for MarkovToken<'a> {
    fn from(owned: &'a MarkovTokenOwned) -> Self {
        match owned {
            MarkovTokenOwned::Textlet(s) => MarkovToken::Textlet(s),
            MarkovTokenOwned::Begin => MarkovToken::Begin,
            MarkovTokenOwned::End => MarkovToken::End,
        }
    }
}

impl<'a> From<&'a MarkovTokenOwned> for &'a str {
    fn from(token: &'a MarkovTokenOwned) -> Self {
        match token {
            MarkovTokenOwned::Textlet(s) => s,
            MarkovTokenOwned::Begin => "",
            MarkovTokenOwned::End => "",
        }
    }
}

impl<'a> From<&'a MarkovToken<'a>> for &'a str {
    fn from(token: &'a MarkovToken<'a>) -> Self {
        token.string_ref()
    }
}

/// A list of [MarkovToken]s; a sentence.
#[derive(Debug)]
pub struct TokenList<'a>(pub LinkedList<MarkovToken<'a>>);

impl<'a> TokenList<'a> {
    pub fn iter(&self) -> std::collections::linked_list::Iter<MarkovToken> {
        self.0.iter()
    }

    pub fn is_empty(&self) -> bool {
        self.iter().any(|x| x.len() > 0)
    }

    pub fn len(&self) -> usize {
        self.iter().map(|x| x.len()).sum()
    }
}

impl<'a> Display for TokenList<'a> {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), std::fmt::Error> {
        for x in &self.0 {
            fmt.write_str(x.string_ref())?;
        }

        Ok(())
    }
}