neatify 0.1.4

A code formatter library for multiple languages with a clean API interface
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

//! Token definitions for code formatting

/// Token types for code formatting
#[derive(Debug, PartialEq, Clone)]
pub enum Token {
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    Semicolon,
    Colon,
    Comma,
    Dot,
    Operator(String),
    Keyword(String),
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(String),
    Comment(String),
    Whitespace(String),
    Newline,
    Other(char),
}

/// Trait for language-specific tokenizers
pub trait Tokenizer: std::panic::RefUnwindSafe {
    /// Tokenize source code into tokens
    fn tokenize(&self, content: &str) -> Vec<Token>;
}

/// Parse source code into tokens
pub fn parse(content: &str, tokenizer: &dyn Tokenizer) -> Vec<Token> {
    tokenizer.tokenize(content)
}