udled 0.8.0

Tokenizer and parser
Documentation
use udled::tokenizers::{Alphabetic, AsciiWhiteSpace, Digit};
use udled::{prelude::*, Char, Input, Span};

#[test]
fn many_one_or_more() {
    let mut input = Input::new("abc123");
    let item = input.parse(Alphabetic.many()).unwrap();
    let chars: String = item.value.into_iter().map(|i| i.value).collect();
    assert_eq!(chars, "abc");
    assert_eq!(item.span, Span::new(0, 3));
}

#[test]
fn many_empty_fails() {
    let mut input = Input::new("");
    assert!(input.parse(Char.many()).is_err());
}

#[test]
fn many_eat() {
    let mut input = Input::new("aaa");
    input.eat('a'.many()).unwrap();
    assert!(input.parse(udled::EOF).is_ok());
}

#[test]
fn optional_some_and_none() {
    let mut input = Input::new("abc");
    let item = input.parse('a'.optional()).unwrap();
    assert_eq!(item.unwrap().value, 'a');

    let mut input = Input::new("abc");
    let item = input.parse('z'.optional()).unwrap();
    assert!(item.is_none());
    // optional should not consume input
    let first = input.parse('a').unwrap();
    assert_eq!(first.value, 'a');
}

#[test]
fn or_combinator() {
    let mut input = Input::new("b");
    let item = input.parse('a'.or('b')).unwrap();
    assert!(matches!(item, udled::Either::Right(_)));

    let mut input = Input::new("c");
    assert!(input.parse('a'.or('b')).is_err());
}

#[test]
fn repeat_exact() {
    let mut input = Input::new("aaaa");
    let item = input.parse('a'.repeat(3)).unwrap();
    assert_eq!(item.value.len(), 3);
    assert_eq!(item.span, Span::new(0, 3));

    let mut input = Input::new("aa");
    assert!(input.parse('a'.repeat(3)).is_err());
}

#[test]
fn until_combinator() {
    use udled::tokenizers::Until;

    let mut input = Input::new("abc;");
    let item = input.parse(Char.until(';')).unwrap();
    let chars: String = item.value.into_iter().map(|i| i.value).collect();
    assert_eq!(chars, "abc");
    assert_eq!(item.span, Span::new(0, 3));

    let mut input = Input::new("abc");
    assert!(input.parse::<Until<_, _, _>>(Char.until(';')).is_err());
}

#[test]
fn punctuated_basic() {
    let mut input = Input::new("1,2,3");
    let list = input
        .parse(udled::tokenizers::Punctuated::new(Digit::default(), ','))
        .unwrap();
    let items: Vec<u32> = list.into_items().map(|i| i.value).collect();
    assert_eq!(items, vec![1, 2, 3]);
}

#[test]
fn punctuated_empty() {
    let mut input = Input::new("");
    let list = input
        .parse(udled::tokenizers::Punctuated::new(Digit::default(), ','))
        .unwrap();
    assert!(list.is_empty());
}

#[test]
fn punctuated_non_empty() {
    let mut input = Input::new("1,2");
    let list = input
        .parse(udled::tokenizers::Punctuated::new(Digit::default(), ',').non_empty())
        .unwrap();
    let items: Vec<u32> = list.into_items().map(|i| i.value).collect();
    assert_eq!(items, vec![1, 2]);

    let mut input = Input::new("");
    assert!(input
        .parse(udled::tokenizers::Punctuated::new(Digit::default(), ',').non_empty(),)
        .is_err());
}

#[test]
fn spanned_combinator() {
    let mut input = Input::new("abc");
    let span = input.parse("ab".spanned()).unwrap();
    assert_eq!(span, Span::new(0, 2));
}

#[test]
fn slice_combinator() {
    let mut input = Input::new("hello");
    let item = input.parse("hell".slice()).unwrap();
    assert_eq!(item.value, "hell");
    assert_eq!(item.span, Span::new(0, 4));
}

#[test]
fn map_ok_combinator() {
    let mut input = Input::new("a");
    let item = input
        .parse(Char.map_ok(|c| c.value.to_uppercase().to_string()))
        .unwrap();
    assert_eq!(item, "A");
}

#[test]
fn map_err_combinator() {
    let mut input = Input::new("b");
    let err = input
        .parse('a'.map_err(|pos, _| format!("expected 'a' at {}", pos)))
        .unwrap_err();
    assert!(err.to_string().contains("expected 'a' at 0"));
}

#[test]
fn into_integer_with_many() {
    let mut input = Input::new("12345");
    let digits: Vec<_> = input
        .parse(Digit::default().many())
        .unwrap()
        .value
        .into_iter()
        .map(|i| i.value)
        .collect();
    assert_eq!(digits, vec![1, 2, 3, 4, 5]);
}

#[test]
fn into_integer_with_tuple() {
    let mut input = Input::new("12");
    let item = input
        .parse((Digit::default(), Digit::default()).into_integer(10))
        .unwrap();
    assert_eq!(item.value, 12i128);
}

#[test]
fn into_integer_hex() {
    let mut input = Input::new("FF");
    let item = input
        .parse((Digit(16), Digit(16)).into_integer(16))
        .unwrap();
    assert_eq!(item.value, 255i128);
}

#[test]
fn complex_expression() {
    let mut input = Input::new("1, 2, 3");
    let list = input
        .parse(Digit::default().punctuated((
            AsciiWhiteSpace.many().optional(),
            ',',
            AsciiWhiteSpace.many().optional(),
        )))
        .unwrap();
    let items: Vec<u32> = list.into_items().map(|i| i.value).collect();
    assert_eq!(items, vec![1, 2, 3]);
}