rust-forth-tokenizer 0.2.0

A Forth tokenizer written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# rust-forth-tokenizer
A Forth tokenizer written in Rust

Usage:

main() {
use rust_forth_tokenizer::ForthToken;
use rust_forth_tokenizer::ForthTokenizer;

     let tokenizer = ForthTokenizer::new("word : wordname 1 2 3 ; definition");
     // The code also supports the regular for loop iterator syntax
     let collected: Vec<_> = tokenizer.into_iter().collect();
     assert_eq!(
            &collected,
            &vec![
                ForthToken::Command("word"),
                ForthToken::Colon,
                ForthToken::Command("wordname"),
                ForthToken::Number(1),
                ForthToken::Number(2),
                ForthToken::Number(3),
                ForthToken::SemiColon,
                ForthToken::Command("definition"),
            ]
        );    
}