use super::*;
use abortable_parser::{Result, SliceIter};
use crate::iter::OffsetStrIter;
#[test]
fn test_empty_token() {
let result = emptytok(OffsetStrIter::new("NULL "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "NULL");
assert_eq!(tok.typ, TokenType::EMPTY);
}
}
#[test]
fn test_assert_token() {
let result = asserttok(OffsetStrIter::new("assert "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "assert");
assert_eq!(tok.typ, TokenType::BAREWORD);
}
}
#[test]
fn test_out_token() {
let result = outtok(OffsetStrIter::new("out "));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "out");
assert_eq!(tok.typ, TokenType::BAREWORD);
}
}
#[test]
fn test_out_token_with_comment() {
let result = outtok(OffsetStrIter::new("out//comment"));
assert!(
result.is_complete(),
format!("result {:?} is not done", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "out");
assert_eq!(tok.typ, TokenType::BAREWORD);
}
}
#[test]
fn test_not_out_token() {
let result = outtok(OffsetStrIter::new("output"));
assert!(result.is_fail(), format!("result {:?} is not fail", result));
}
#[test]
fn test_escape_quoted() {
let result = escapequoted(OffsetStrIter::new("foo \\\"bar\""));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
);
if let Result::Complete(_rest, frag) = result {
assert_eq!(frag, "foo \"bar");
}
}
#[test]
fn test_string_with_escaping() {
let result = strtok(OffsetStrIter::new("\"foo \\\\ \\\"bar\""));
assert!(
result.is_complete(),
format!("result {:?} is not ok", result)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.fragment, "foo \\ \"bar".to_string());
}
}
#[test]
fn test_tokenize_bareword_with_dash() {
let input = OffsetStrIter::new("foo-bar ");
let result = tokenize(input.clone());
assert!(result.is_ok(), format!("result {:?} is not ok", result));
if let Ok(toks) = result {
assert_eq!(toks.len(), 2);
assert_eq!(toks[0].fragment, "foo-bar");
}
}
macro_rules! assert_token {
($input:expr, $typ:expr, $msg:expr) => {
let result = token(OffsetStrIter::new($input));
assert!(
result.is_complete(),
format!("result {:?} is not a {}", result, $msg)
);
if let Result::Complete(_, tok) = result {
assert_eq!(tok.typ, $typ);
assert_eq!(tok.fragment, $input);
}
};
}
#[test]
fn test_digittok() {
assert_token!("1", TokenType::DIGIT, "1");
}
#[test]
fn test_boolean() {
assert_token!("true", TokenType::BOOLEAN, "boolean");
}
#[test]
fn test_eqeqtok() {
assert_token!("==", TokenType::PUNCT, "==");
}
#[test]
fn test_notequaltok() {
assert_token!("!=", TokenType::PUNCT, "!=");
}
#[test]
fn test_gttok() {
assert_token!(">", TokenType::PUNCT, ">");
}
#[test]
fn test_lttok() {
assert_token!("<", TokenType::PUNCT, "<");
}
#[test]
fn test_gteqtok() {
assert_token!(">=", TokenType::PUNCT, ">=");
}
#[test]
fn test_lteqtok() {
assert_token!("<=", TokenType::PUNCT, "<=");
}
#[test]
fn test_tokenize_one_of_each() {
let input = OffsetStrIter::new(
"map out filter assert let import func select as => [ ] { } ; = % / * \
+ - . ( ) , 1 . foo \"bar\" // comment\n ; true false == < > <= >= !=",
);
let result = tokenize(input.clone());
assert!(result.is_ok(), format!("result {:?} is not ok", result));
let v = result.unwrap();
for (i, t) in v.iter().enumerate() {
println!("{}: {:?}", i, t);
}
assert_eq!(v.len(), 39);
assert_eq!(v[38].typ, TokenType::END);
}
#[test]
fn test_parse_has_end() {
let input = OffsetStrIter::new("foo");
let result = tokenize(input.clone());
assert!(result.is_ok());
let v = result.unwrap();
assert_eq!(v.len(), 2);
assert_eq!(v[1].typ, TokenType::END);
}
#[test]
fn test_whitespace() {
assert!(whitespace(OffsetStrIter::new(" ")).is_complete());
let result = whitespace(OffsetStrIter::new(" "));
match result {
Result::Complete(rest, o) => {
assert_eq!(rest.get_offset(), 2);
assert_eq!(o.typ, TokenType::WS);
}
_ => assert!(false, "Not complete"),
}
}
#[test]
fn test_parse_comment() {
assert!(comment(OffsetStrIter::new("// comment\n")).is_complete());
assert!(comment(OffsetStrIter::new("// comment")).is_complete());
let mut parsed = comment(OffsetStrIter::new("// comment\n"));
assert!(parsed.is_complete());
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position {
file: None,
line: 1,
column: 1,
offset: 0
},
}
);
}
assert!(comment(OffsetStrIter::new("// comment\r\n")).is_complete());
parsed = comment(OffsetStrIter::new("// comment\r\n"));
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position {
file: None,
column: 1,
line: 1,
offset: 0
},
}
);
}
assert!(comment(OffsetStrIter::new("// comment\r\n ")).is_complete());
parsed = comment(OffsetStrIter::new("// comment\r\n "));
if let Result::Complete(_rest, cmt) = parsed {
assert_eq!(
cmt,
Token {
typ: TokenType::COMMENT,
fragment: " comment".to_string(),
pos: Position {
file: None,
column: 1,
line: 1,
offset: 0
},
}
);
}
assert!(comment(OffsetStrIter::new("// comment")).is_complete());
}
#[test]
fn test_match_word() {
let input = vec![Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position {
file: None,
line: 1,
column: 1,
offset: 0,
},
}];
let result = word!(SliceIter::new(input.as_slice()), "foo");
match result {
Result::Complete(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
#[test]
fn test_match_word_empty_input() {
let input = vec![Token {
fragment: "".to_string(),
typ: TokenType::END,
pos: Position {
file: None,
line: 1,
column: 1,
offset: 0,
},
}];
let result = word!(SliceIter::new(input.as_slice()), "foo");
match result {
Result::Complete(_, _) => assert!(false, "Should have been an error but was Done"),
Result::Incomplete(_) => assert!(false, "Should have been a Fail but was Incomplete"),
Result::Fail(_) => {
}
Result::Abort(_) => assert!(false, "Should have been a Fail but was Abort"),
}
}
#[test]
fn test_match_punct() {
let input = vec![Token {
fragment: "!".to_string(),
typ: TokenType::PUNCT,
pos: Position {
file: None,
line: 1,
column: 1,
offset: 0,
},
}];
let result = punct!(SliceIter::new(input.as_slice()), "!");
match result {
Result::Complete(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}
#[test]
fn test_match_type() {
let input = vec![Token {
fragment: "foo".to_string(),
typ: TokenType::BAREWORD,
pos: Position {
file: None,
line: 1,
column: 1,
offset: 0,
},
}];
let result = match_type!(SliceIter::new(input.as_slice()), BAREWORD);
match result {
Result::Complete(_, tok) => assert_eq!(tok, input[0]),
res => assert!(false, format!("Fail: {:?}", res)),
}
}