Enum tokenizer_py::Token

pub enum Token {
    EndMarker,
    Name(String),
    Number(String),
    String(String),
    NewLine,
    OP(String),
    Indent(String),
    Dedent,
    Comment(String),
    NL,
}

An enumeration of Python tokens.

Example

use tokenizer_py::{Tokenizer, Token};

struct BinaryExp{
    left: Token, center: Token,right: Token,
}
impl BinaryExp {
    fn new(left: Token, center: Token, right: Token) -> Self {
        BinaryExp { left, center, right}
    }
    fn execute(&self) -> Result<isize, <isize as std::str::FromStr>::Err> {
        use Token::{Number, OP};
        match (&self.left, &self.center, &self.right) {
            (Number(ref left), OP(ref op), Number(ref right)) => match op.as_str() {
                "+" => Ok(left.parse::<isize>()? + right.parse::<isize>()?),
                "-" => Ok(left.parse::<isize>()? - right.parse::<isize>()?),
                "*" => Ok(left.parse::<isize>()? * right.parse::<isize>()?),
                "/" => Ok(left.parse::<isize>()? / right.parse::<isize>()?),
                "%" => Ok(left.parse::<isize>()? % right.parse::<isize>()?),
                _ => panic!("Invalid operator"),
            }
            _ => panic!("Invalid tokens"),
        }
    }
}

let tokenizer = Tokenizer::new("10 + 10".to_owned());
let mut tokens = tokenizer.tokenize().unwrap();
let _ = tokens.pop(); // remove EndMarker

let binexp = BinaryExp::new(
    tokens.pop().unwrap(),
    tokens.pop().unwrap(),
    tokens.pop().unwrap()
);

assert_eq!(binexp.execute(), Ok(20));

Variants

EndMarker

Indicates the end of the program.

Name(String)

A name token, such as a function or variable name.

Number(String)

A number token, such as a literal integer or floating-point number.

String(String)

A string token, such as a single or double-quoted string.

NewLine

A newline token, indicating a new line in the source code.

OP(String)

An operator token, such as an arithmetic or comparison operator.

Indent(String)

An indent token, indicating that a block of code is being indented.

Dedent

A dedent token, indicating that a block of code is being dedented.

Comment(String)

A comment token, such as a single-line or multi-line comment.

NL

A token indicating a new line, for compatibility with the original tokenizer.

Trait Implementations

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl PartialEq for Token

fn eq(&self, other: &Token) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Eq for Token

impl StructuralPartialEq for Token