doublify_toolkit/filtering/
tokenizer.rs

1// The MIT License (MIT)
2//
3// Copyright (c) 2017 Doublify Technologies
4//
5// Permission is hereby granted, free of charge, to any person obtaining a copy
6// of this software and associated documentation files (the "Software"), to deal
7// in the Software without restriction, including without limitation the rights
8// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9// copies of the Software, and to permit persons to whom the Software is
10// furnished to do so, subject to the following conditions:
11//
12// The above copyright notice and this permission notice shall be included in
13// all copies or substantial portions of the Software.
14//
15// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21// THE SOFTWARE.
22
23use filtering::scanner::{Kind, Token, scan};
24
25fn is_quote(ch: char) -> bool {
26  ch == '"' || ch == '\''
27}
28
29trait Matching {
30  fn is_boolean(&self) -> bool;
31  fn is_integer(&self) -> bool;
32  fn is_string(&self) -> bool;
33}
34
35impl Matching for Token {
36  fn is_boolean(&self) -> bool {
37    match self.binding.as_ref() {
38      "true" | "false" => true,
39      _ => false,
40    }
41  }
42
43  fn is_integer(&self) -> bool {
44    self.binding.parse::<f64>().is_ok()
45  }
46
47  fn is_string(&self) -> bool {
48    let beginning = self.binding.chars().nth(0).unwrap();
49    let ending = self.binding.chars().last().unwrap();
50
51    is_quote(beginning) && beginning == ending
52  }
53}
54
55fn match_identifier(raw_token: &Token) -> Token {
56  let token = raw_token.change_kind(
57    match raw_token {
58      ref t if t.is_string() => Kind::String,
59      ref t if t.is_integer() => Kind::Integer,
60      ref t if t.is_boolean() => Kind::Boolean,
61      _ => raw_token.kind,
62    },
63  );
64
65  token
66}
67
68fn match_token(raw_token: &Token) -> Token {
69  match raw_token.kind {
70    Kind::Identifier => match_identifier(raw_token),
71    _ => raw_token.to_owned(),
72  }
73}
74
75/// Tokenizes query
76pub fn tokenize(raw_query: &str) -> Vec<Token> {
77  let tokens = scan(raw_query);
78
79  tokens.iter().map(match_token).collect()
80}
doublify_toolkit/filtering/tokenizer.rs

doublify_toolkit/filtering/
tokenizer.rs
