context_forge/analysis/
lexicon.rs

1/// Configurable lexicon sets for category detection.
2#[derive(Debug, Clone)]
3#[non_exhaustive]
4pub struct Lexicons {
5    /// Words/phrases indicating negation or contradiction (e.g. "not", "no longer").
6    pub negation_markers: Vec<String>,
7    /// Words/phrases indicating comparison (e.g. "instead of", "rather than").
8    pub comparison_markers: Vec<String>,
9    /// Words/phrases indicating causal reasoning (e.g. "because", "so that").
10    pub causal_connectors: Vec<String>,
11    /// Words/phrases indicating confirmation (e.g. "confirmed", "agreed").
12    pub confirmation_tokens: Vec<String>,
13    /// Words/phrases indicating a state-change operation (e.g. "is now", "set to").
14    pub state_operators: Vec<String>,
15}
16
17impl Default for Lexicons {
18    fn default() -> Self {
19        Self {
20            negation_markers: vec![
21                "not".to_string(),
22                "no".to_string(),
23                "never".to_string(),
24                "neither".to_string(),
25                "nor".to_string(),
26                "don't".to_string(),
27                "doesn't".to_string(),
28                "didn't".to_string(),
29                "won't".to_string(),
30                "wouldn't".to_string(),
31                "can't".to_string(),
32                "cannot".to_string(),
33                "couldn't".to_string(),
34                "shouldn't".to_string(),
35                "isn't".to_string(),
36                "aren't".to_string(),
37                "wasn't".to_string(),
38                "weren't".to_string(),
39                "nothing".to_string(),
40                "nowhere".to_string(),
41                "nobody".to_string(),
42                "none".to_string(),
43                "without".to_string(),
44            ],
45            comparison_markers: vec![
46                "better".to_string(),
47                "worse".to_string(),
48                "over".to_string(),
49                "instead".to_string(),
50                "rather".to_string(),
51                "prefer".to_string(),
52                "chose".to_string(),
53                "picked".to_string(),
54                "replaced".to_string(),
55                "switched".to_string(),
56                "upgraded".to_string(),
57                "downgraded".to_string(),
58                "versus".to_string(),
59                "vs".to_string(),
60                "compared".to_string(),
61                "alternative".to_string(),
62                "option".to_string(),
63                "decided".to_string(),
64                "went with".to_string(),
65                "selected".to_string(),
66            ],
67            causal_connectors: vec![
68                "because".to_string(),
69                "since".to_string(),
70                "due to".to_string(),
71                "caused by".to_string(),
72                "reason".to_string(),
73                "therefore".to_string(),
74                "so that".to_string(),
75                "as a result".to_string(),
76                "which means".to_string(),
77                "in order to".to_string(),
78                "leads to".to_string(),
79                "that's why".to_string(),
80                "the reason is".to_string(),
81                "this is because".to_string(),
82            ],
83            confirmation_tokens: vec![
84                "yes".to_string(),
85                "perfect".to_string(),
86                "correct".to_string(),
87                "exactly".to_string(),
88                "good".to_string(),
89                "right".to_string(),
90                "agreed".to_string(),
91                "confirmed".to_string(),
92                "approved".to_string(),
93                "lgtm".to_string(),
94            ],
95            state_operators: vec![
96                "=".to_string(),
97                ":=".to_string(),
98                "set to".to_string(),
99                "changed to".to_string(),
100                "now uses".to_string(),
101                "updated to".to_string(),
102                "IP:".to_string(),
103                "port:".to_string(),
104                "is now".to_string(),
105                "is set to".to_string(),
106                "is changed to".to_string(),
107                "is updated to".to_string(),
108                "→".to_string(),
109                "->".to_string(),
110            ],
111        }
112    }
113}
114
115#[cfg(test)]
116mod tests {
117    use super::Lexicons;
118
119    #[test]
120    fn default_lexicons_have_expected_counts() {
121        let lexicons = Lexicons::default();
122
123        assert_eq!(lexicons.negation_markers.len(), 23);
124        assert_eq!(lexicons.comparison_markers.len(), 20);
125        assert_eq!(lexicons.causal_connectors.len(), 14);
126        assert_eq!(lexicons.confirmation_tokens.len(), 10);
127        assert_eq!(lexicons.state_operators.len(), 14);
128    }
129
130    #[test]
131    fn state_operators_do_not_contain_bare_is() {
132        let lexicons = Lexicons::default();
133
134        assert!(lexicons
135            .state_operators
136            .iter()
137            .all(|operator| operator.to_lowercase() != "is"));
138    }
139}
context_forge/analysis/lexicon.rs

context_forge/analysis/
lexicon.rs
