1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
//! Erlang source code tokenizer. //! //! # Examples //! //! Tokenizes the Erlang code `io:format("Hello").`: //! //! ``` //! use erl_tokenize::{Tokenizer, TokenKind}; //! //! let src = r#"io:format("Hello")."#; //! let tokenizer = Tokenizer::new(src); //! let tokens = tokenizer.collect::<Result<Vec<(_, _)>, _>>().unwrap(); //! //! assert_eq!(tokens.iter().map(|&(ref t, _)| t.kind()).collect::<Vec<_>>(), //! [TokenKind::Atom, TokenKind::Symbol, TokenKind::Atom, TokenKind::Symbol, //! TokenKind::String, TokenKind::Symbol, TokenKind::Symbol]); //! //! assert_eq!(tokens.iter().map(|&(ref t, _)| t.text()).collect::<Vec<_>>(), //! ["io", ":", "format", "(", r#""Hello""#, ")", "."]); //! ``` //! //! # References //! //! - [erl_scan][erl_scan] module //! - [Erlang Data Types][Data Types] //! //! [erl_scan]: http://erlang.org/doc/man/erl_scan.html //! [Data Types]: http://erlang.org/doc/reference_manual/data_types.html #![warn(missing_docs)] extern crate num; #[macro_use] extern crate trackable; pub use error::{Error, ErrorKind}; pub use token::{Token, TokenKind, TokenValue}; pub use position::Position; pub use tokenizer::Tokenizer; pub mod tokens; pub mod values; mod error; mod token; mod position; mod tokenizer; mod util; /// This crate specific `Result` type. pub type Result<T> = ::std::result::Result<T, Error>; #[cfg(test)] mod tests { #[test] fn it_works() {} }