polyglot_tokenizer 0.2.1

A generic programming language tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

pub mod tokenizer;
pub use tokenizer::{Token, Tokenizer};

/// Tokenize the content and return only the identifiers and symbols from the langauge
///
/// # Examples
/// ```
/// use polyglot_tokenizer;
/// let content = r#"let x = [5, "hello"];"#;
/// let tokens: Vec<&str> = polyglot_tokenizer::get_key_tokens(content).collect();
/// assert_eq!(tokens, vec!["let", "x", "=", "[", ",", "]", ";"]);
/// ```
pub fn get_key_tokens(content: &str) -> impl Iterator<Item = &str> {
    Tokenizer::new(content).tokens().filter_map(|t| match t {
        Token::Ident(t) | Token::Symbol(t) => Some(t),
        _ => None,
    })
}