use crate::data::is_boundary;
use crate::{Error, ErrorKind, Scalar};
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
pub enum Operator {
LessThan,
LessThanEqual,
GreaterThan,
GreaterThanEqual,
}
#[derive(Debug, PartialEq)]
pub enum TextToken<'a> {
Array(usize),
Object(usize),
HiddenObject(usize),
Scalar(Scalar<'a>),
Operator(Operator),
End(usize),
Header(Scalar<'a>),
}
#[derive(Debug, Default)]
pub struct TextTapeParser;
impl TextTapeParser {
pub fn new() -> Self {
TextTapeParser::default()
}
pub fn parse_slice(self, data: &[u8]) -> Result<TextTape, Error> {
let mut res = TextTape::default();
self.parse_slice_into_tape(data, &mut res)?;
Ok(res)
}
pub fn parse_slice_into_tape<'a>(
self,
data: &'a [u8],
tape: &mut TextTape<'a>,
) -> Result<(), Error> {
let token_tape = &mut tape.token_tape;
token_tape.clear();
token_tape.reserve(data.len() / 5);
let mut state = ParserState {
data,
original_length: data.len(),
token_tape,
};
state.parse()?;
Ok(())
}
}
struct ParserState<'a, 'b> {
data: &'a [u8],
original_length: usize,
token_tape: &'b mut Vec<TextToken<'a>>,
}
#[derive(Debug, Default)]
pub struct TextTape<'a> {
token_tape: Vec<TextToken<'a>>,
}
#[derive(Debug, PartialEq)]
enum ParseState {
Key,
KeyValueSeparator,
ObjectValue,
ArrayValue,
ParseOpen,
FirstValue,
EmptyObject,
}
fn parse_quote_scalar_fallback(d: &[u8]) -> Result<(Scalar, &[u8]), Error> {
let mut pos = 1;
while pos < d.len() {
if d[pos] == b'\\' {
pos += 2;
} else if d[pos] == b'"' {
let scalar = Scalar::new(&d[1..pos]);
return Ok((scalar, &d[pos + 1..]));
} else {
pos += 1;
}
}
Err(Error::eof())
}
#[cfg(not(target_arch = "x86_64"))]
fn parse_quote_scalar(d: &[u8]) -> Result<(Scalar, &[u8]), Error> {
use crate::util::{contains_zero_byte, repeat_byte};
let sd = &d[1..];
unsafe {
let start_ptr = sd.as_ptr();
let end_ptr = start_ptr.add(sd.len() / 8 * 8);
let mut ptr = start_ptr;
while ptr < end_ptr {
let acc = (ptr as *const u64).read_unaligned();
if contains_zero_byte(acc ^ repeat_byte(b'\\')) {
break;
} else if contains_zero_byte(acc ^ repeat_byte(b'"')) {
while *ptr != b'"' {
ptr = ptr.offset(1);
}
let offset = sub(ptr, start_ptr);
let (scalar, rest) = sd.split_at(offset);
let s = Scalar::new(scalar);
return Ok((s, &rest[1..]));
}
ptr = ptr.offset(8);
}
}
parse_quote_scalar_fallback(d)
}
#[cfg(target_arch = "x86_64")]
fn parse_quote_scalar(d: &[u8]) -> Result<(Scalar, &[u8]), Error> {
#[target_feature(enable = "sse2")]
unsafe fn inner(d: &[u8]) -> Result<(Scalar, &[u8]), Error> {
use core::arch::x86_64::*;
let haystack = &d[1..];
let start_ptr = haystack.as_ptr();
let mut ptr = start_ptr;
let loop_size = std::mem::size_of::<__m128i>();
let end_ptr = haystack[haystack.len()..].as_ptr().sub(loop_size);
let quote = _mm_set1_epi8(b'"' as i8);
let slash = _mm_set1_epi8(b'\\' as i8);
while ptr <= end_ptr {
let reg = _mm_loadu_si128(ptr as *const __m128i);
let slash_found = _mm_cmpeq_epi8(slash, reg);
if _mm_movemask_epi8(slash_found) != 0 {
break;
}
let quote_found = _mm_cmpeq_epi8(quote, reg);
let mask = _mm_movemask_epi8(quote_found);
if mask != 0 {
let at = sub(ptr, start_ptr);
let end_idx = at + (mask.trailing_zeros() as usize);
let scalar = std::slice::from_raw_parts(start_ptr, end_idx);
let scalar = Scalar::new(scalar);
return Ok((scalar, &haystack[end_idx + 1..]));
}
ptr = ptr.add(loop_size);
}
parse_quote_scalar_fallback(d)
}
unsafe { inner(&d) }
}
#[inline]
fn split_at_scalar_fallback(d: &[u8]) -> (Scalar, &[u8]) {
let start_ptr = d.as_ptr();
let end_ptr = unsafe { start_ptr.add(d.len()) };
let nind = unsafe { forward_search(start_ptr, end_ptr, is_boundary) };
let mut ind = nind.unwrap_or_else(|| d.len());
ind = std::cmp::max(ind, 1);
let (scalar, rest) = d.split_at(ind);
(Scalar::new(scalar), rest)
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
fn split_at_scalar(d: &[u8]) -> (Scalar, &[u8]) {
split_at_scalar_fallback(d)
}
#[cfg(target_arch = "x86_64")]
#[inline]
fn split_at_scalar(d: &[u8]) -> (Scalar, &[u8]) {
#[target_feature(enable = "sse2")]
#[inline]
#[allow(overflowing_literals)]
unsafe fn inner(d: &[u8]) -> (Scalar, &[u8]) {
use core::arch::x86_64::*;
let start_ptr = d.as_ptr();
let loop_size = std::mem::size_of::<__m128i>();
let end_ptr = d[d.len()..].as_ptr().sub(loop_size);
let mut ptr = start_ptr;
while ptr <= end_ptr {
let input = _mm_loadu_si128(ptr as *const __m128i);
let t0 = _mm_cmpeq_epi8(input, _mm_set1_epi8(9));
let mut result = t0;
let t1 = _mm_cmpeq_epi8(input, _mm_set1_epi8(10));
result = _mm_or_si128(result, t1);
let t2 = _mm_cmpeq_epi8(input, _mm_set1_epi8(13));
result = _mm_or_si128(result, t2);
let t3 = _mm_cmpeq_epi8(input, _mm_set1_epi8(32));
result = _mm_or_si128(result, t3);
let t4 = _mm_cmpeq_epi8(input, _mm_set1_epi8(35));
result = _mm_or_si128(result, t4);
let t5 = _mm_cmpeq_epi8(input, _mm_set1_epi8(60));
result = _mm_or_si128(result, t5);
let t6 = _mm_cmpeq_epi8(input, _mm_set1_epi8(61));
result = _mm_or_si128(result, t6);
let t7 = _mm_cmpeq_epi8(input, _mm_set1_epi8(62));
result = _mm_or_si128(result, t7);
let t8 = _mm_cmpeq_epi8(input, _mm_set1_epi8(123));
result = _mm_or_si128(result, t8);
let t9 = _mm_cmpeq_epi8(input, _mm_set1_epi8(125));
result = _mm_or_si128(result, t9);
let found_mask = _mm_movemask_epi8(result);
if found_mask != 0 {
let at = sub(ptr, start_ptr);
let end_idx = at + (found_mask.trailing_zeros() as usize);
let end_idx = std::cmp::max(end_idx, 1);
let scalar = std::slice::from_raw_parts(start_ptr, end_idx);
let scalar = Scalar::new(scalar);
return (scalar, &d[end_idx..]);
}
ptr = ptr.add(loop_size);
}
split_at_scalar_fallback(d)
}
unsafe { inner(&d) }
}
impl<'a> TextTape<'a> {
pub fn new() -> Self {
Default::default()
}
pub fn from_slice(data: &[u8]) -> Result<TextTape<'_>, Error> {
TextTapeParser.parse_slice(data)
}
pub fn parser() -> TextTapeParser {
TextTapeParser
}
pub fn tokens(&self) -> &[TextToken<'a>] {
self.token_tape.as_slice()
}
}
impl<'a, 'b> ParserState<'a, 'b> {
fn offset(&self, data: &[u8]) -> usize {
self.original_length - data.len()
}
#[inline]
fn skip_ws_t(&mut self, data: &'a [u8]) -> Option<&'a [u8]> {
unsafe {
let start_ptr = data.as_ptr();
let end_ptr = start_ptr.add(data.len());
let mut ptr = start_ptr;
while ptr < end_ptr {
match *ptr {
b' ' | b'\t' | b'\n' | b'\r' => {}
b'#' => {
ptr = ptr.offset(1);
while ptr < end_ptr && *ptr != b'\n' {
ptr = ptr.offset(1);
}
}
_ => {
let rest = std::slice::from_raw_parts(ptr, sub(end_ptr, ptr));
return Some(rest);
}
}
ptr = ptr.offset(1);
}
}
None
}
#[inline]
fn parse_quote_scalar(&mut self, d: &'a [u8]) -> Result<&'a [u8], Error> {
let (scalar, rest) = parse_quote_scalar(d)?;
self.token_tape.push(TextToken::Scalar(scalar));
Ok(rest)
}
#[inline]
fn parse_scalar(&mut self, d: &'a [u8]) -> &'a [u8] {
let (scalar, rest) = split_at_scalar(d);
self.token_tape.push(TextToken::Scalar(scalar));
rest
}
#[inline]
fn parse_key_value_separator(&mut self, d: &'a [u8]) -> &'a [u8] {
if d[0] == b'=' {
&d[1..]
} else if d[0] == b'<' {
if d.get(1).map_or(false, |c| *c == b'=') {
self.token_tape
.push(TextToken::Operator(Operator::LessThanEqual));
&d[2..]
} else {
self.token_tape
.push(TextToken::Operator(Operator::LessThan));
&d[1..]
}
} else if d[0] == b'>' {
if d.get(1).map_or(false, |c| *c == b'=') {
self.token_tape
.push(TextToken::Operator(Operator::GreaterThanEqual));
&d[2..]
} else {
self.token_tape
.push(TextToken::Operator(Operator::GreaterThan));
&d[1..]
}
} else {
d
}
}
#[inline]
pub fn parse(&mut self) -> Result<(), Error> {
let mut data = self.data;
let mut state = ParseState::Key;
let mut array_ind_of_hidden_obj = None;
let mut parent_ind = 0;
loop {
let d = match self.skip_ws_t(data) {
Some(d) => d,
None => {
if parent_ind == 0 && state == ParseState::Key {
return Ok(());
} else {
return Err(Error::eof());
}
}
};
data = d;
match state {
ParseState::EmptyObject => {
if data[0] != b'}' {
return Err(Error::new(ErrorKind::InvalidEmptyObject {
offset: self.offset(data),
}));
}
data = &data[1..];
state = ParseState::Key;
}
ParseState::Key => {
match data[0] {
b'}' => {
let grand_ind = match self.token_tape.get(parent_ind) {
Some(TextToken::Array(x)) => *x,
Some(TextToken::Object(x)) => *x,
_ => 0,
};
state = match self.token_tape.get(grand_ind) {
Some(TextToken::Array(_x)) => ParseState::ArrayValue,
Some(TextToken::Object(_x)) => ParseState::Key,
_ => ParseState::Key,
};
let end_idx = self.token_tape.len();
if parent_ind == 0 && grand_ind == 0 {
return Err(Error::new(ErrorKind::StackEmpty {
offset: self.offset(data),
}));
}
self.token_tape.push(TextToken::End(parent_ind));
if let Some(array_ind) = array_ind_of_hidden_obj.take() {
self.token_tape[parent_ind] = TextToken::HiddenObject(end_idx);
let end_idx = self.token_tape.len();
self.token_tape.push(TextToken::End(array_ind));
let grand_ind =
if let Some(parent) = self.token_tape.get_mut(array_ind) {
let grand_ind = match parent {
TextToken::Array(x) => *x,
_ => 0,
};
*parent = TextToken::Array(end_idx);
grand_ind
} else {
0
};
state = match self.token_tape.get(grand_ind) {
Some(TextToken::Array(_x)) => ParseState::ArrayValue,
Some(TextToken::Object(_x)) => ParseState::Key,
_ => ParseState::Key,
};
parent_ind = grand_ind;
} else {
self.token_tape[parent_ind] = TextToken::Object(end_idx);
parent_ind = grand_ind;
}
data = &data[1..];
}
b'{' => {
data = &data[1..];
if let Some(last) = self.token_tape.last_mut() {
if let TextToken::Scalar(x) = last {
if array_ind_of_hidden_obj.is_some() {
return Err(Error::new(ErrorKind::InvalidSyntax {
offset: self.offset(data) - 2,
msg: String::from(
"header values inside a hidden object are unsupported",
),
}));
}
*last = TextToken::Header(*x);
self.token_tape.push(TextToken::Array(0));
state = ParseState::ParseOpen;
} else {
state = ParseState::EmptyObject;
}
} else {
state = ParseState::EmptyObject;
}
}
b'"' => {
data = self.parse_quote_scalar(data)?;
state = ParseState::KeyValueSeparator;
}
_ => {
data = self.parse_scalar(data);
state = ParseState::KeyValueSeparator;
}
}
}
ParseState::KeyValueSeparator => {
data = self.parse_key_value_separator(data);
state = ParseState::ObjectValue;
}
ParseState::ObjectValue => {
match data[0] {
b'{' => {
if let Some(array_ind) = array_ind_of_hidden_obj.take() {
let mut start = self.token_tape.len() - 3;
while start > array_ind {
match self.token_tape[start] {
TextToken::End(x) if x == start - 1 => {
start -= 2;
}
_ => {
return Err(Error::new(ErrorKind::InvalidSyntax {
offset: self.offset(data) - 2,
msg: String::from(
"header values inside a hidden object are unsupported",
),
}));
}
}
}
let empty_objects_to_remove = self.token_tape.len() - 2 - array_ind;
let grand_ind = match self.token_tape[array_ind] {
TextToken::Array(x) => x,
_ => 0,
};
for _ in 0..empty_objects_to_remove {
self.token_tape.remove(self.token_tape.len() - 3);
}
parent_ind = array_ind;
self.token_tape[parent_ind] = TextToken::Object(grand_ind);
}
self.token_tape.push(TextToken::Array(0));
state = ParseState::ParseOpen;
data = &data[1..];
}
b'}' => {
state = ParseState::Key;
}
b'"' => {
data = self.parse_quote_scalar(data)?;
state = ParseState::Key;
}
_ => {
data = self.parse_scalar(data);
state = ParseState::Key
}
}
}
ParseState::ParseOpen => {
match data[0] {
b'}' => {
let ind = self.token_tape.len() - 1;
state = match self.token_tape.get(parent_ind) {
Some(TextToken::Array(_x)) => ParseState::ArrayValue,
Some(TextToken::Object(_x)) => ParseState::Key,
_ => ParseState::Key,
};
self.token_tape[ind] = TextToken::Array(ind + 1);
self.token_tape.push(TextToken::End(ind));
data = &data[1..];
}
b'{' => {
let ind = self.token_tape.len() - 1;
self.token_tape[ind] = TextToken::Array(parent_ind);
parent_ind = ind;
state = ParseState::ArrayValue;
}
b'"' => {
data = self.parse_quote_scalar(data)?;
state = ParseState::FirstValue;
}
_ => {
data = self.parse_scalar(data);
state = ParseState::FirstValue;
}
}
}
ParseState::FirstValue => match data[0] {
b'=' => {
let ind = self.token_tape.len() - 2;
self.token_tape[ind] = TextToken::Object(parent_ind);
data = &data[1..];
parent_ind = ind;
state = ParseState::ObjectValue;
}
_ => {
let ind = self.token_tape.len() - 2;
self.token_tape[ind] = TextToken::Array(parent_ind);
parent_ind = ind;
state = ParseState::ArrayValue;
}
},
ParseState::ArrayValue => match data[0] {
b'{' => {
self.token_tape.push(TextToken::Array(0));
state = ParseState::ParseOpen;
data = &data[1..];
}
b'}' => {
let grand_ind = match self.token_tape.get(parent_ind) {
Some(TextToken::Array(x)) => *x,
Some(TextToken::Object(x)) => *x,
_ => 0,
};
state = match self.token_tape.get(grand_ind) {
Some(TextToken::Array(_x)) => ParseState::ArrayValue,
Some(TextToken::Object(_x)) => ParseState::Key,
_ => ParseState::Key,
};
let end_idx = self.token_tape.len();
self.token_tape[parent_ind] = TextToken::Array(end_idx);
self.token_tape.push(TextToken::End(parent_ind));
parent_ind = grand_ind;
data = &data[1..];
}
b'"' => {
data = self.parse_quote_scalar(data)?;
state = ParseState::ArrayValue;
}
b'=' => {
if self.token_tape.len() - parent_ind <= 1
|| matches!(
self.token_tape[self.token_tape.len() - 1],
TextToken::End(_)
)
{
return Err(Error::new(ErrorKind::InvalidSyntax {
msg: String::from("hidden object must start with a key"),
offset: self.offset(data) - 1,
}));
}
let hidden_object = TextToken::Object(parent_ind);
array_ind_of_hidden_obj = Some(parent_ind);
parent_ind = self.token_tape.len() - 1;
self.token_tape
.insert(self.token_tape.len() - 1, hidden_object);
state = ParseState::ObjectValue;
data = &data[1..];
}
_ => {
data = self.parse_scalar(data);
state = ParseState::ArrayValue;
}
},
}
}
}
}
fn sub(a: *const u8, b: *const u8) -> usize {
debug_assert!(a >= b);
(a as usize) - (b as usize)
}
#[inline(always)]
unsafe fn forward_search<F: Fn(u8) -> bool>(
start_ptr: *const u8,
end_ptr: *const u8,
confirm: F,
) -> Option<usize> {
let mut ptr = start_ptr;
while ptr < end_ptr {
if confirm(*ptr) {
return Some(sub(ptr, start_ptr));
}
ptr = ptr.offset(1);
}
None
}
#[cfg(test)]
mod tests {
use super::*;
fn parse<'a>(data: &'a [u8]) -> Result<TextTape<'a>, Error> {
TextTape::from_slice(data)
}
#[test]
fn test_size_of_binary_token() {
assert_eq!(std::mem::size_of::<TextToken>(), 24);
}
#[test]
fn test_binary_tokens_dont_need_to_be_dropped() {
assert!(!std::mem::needs_drop::<TextToken>());
}
#[test]
fn test_simple_event() {
let data = b"foo=bar";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"bar")),
]
);
}
#[test]
fn test_error_offset() {
let data = b"foo={}} a=c";
let err = TextTape::from_slice(&data[..]).unwrap_err();
match err.kind() {
ErrorKind::StackEmpty { offset, .. } => {
assert_eq!(*offset, 6);
}
_ => assert!(false),
}
}
#[test]
fn test_simple_event_with_spaces() {
let data = b" \t\t foo =bar \r\ndef=\tqux";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"def")),
TextToken::Scalar(Scalar::new(b"qux")),
]
);
}
#[test]
fn test_scalars_with_quotes() {
let data = br#""foo"="bar" "3"="1444.11.11""#;
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"3")),
TextToken::Scalar(Scalar::new(b"1444.11.11")),
]
);
}
#[test]
fn test_escaped_quotes() {
let data = br#"name = "Joe \"Captain\" Rogers""#;
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(br#"Joe \"Captain\" Rogers"#)),
]
);
}
#[test]
fn test_escaped_quotes_short() {
let data = br#"name = "J Rogers \"a""#;
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(br#"J Rogers \"a"#)),
]
);
}
#[test]
fn test_escaped_quotes_crazy() {
let data = br#"custom_name="THE !@#$%^&*( '\"LEGION\"')""#;
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"custom_name")),
TextToken::Scalar(Scalar::new(br#"THE !@#$%^&*( '\"LEGION\"')"#)),
]
);
}
#[test]
fn test_numbers_are_scalars() {
let data = b"foo=1.000";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"1.000")),
]
);
}
#[test]
fn test_object_event() {
let data = b"foo={bar=qux}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Object(4),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"qux")),
TextToken::End(1),
]
);
}
#[test]
fn test_object_multi_field_event() {
let data = b"foo={bar=1 qux=28}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Object(6),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"1")),
TextToken::Scalar(Scalar::new(b"qux")),
TextToken::Scalar(Scalar::new(b"28")),
TextToken::End(1),
]
);
}
#[test]
fn test_text_parser_tape() {
let mut tape = TextTape::new();
let data = b"foo={bar=1 qux=28}";
TextTape::parser()
.parse_slice_into_tape(data, &mut tape)
.unwrap();
assert_eq!(
tape.tokens(),
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Object(6),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"1")),
TextToken::Scalar(Scalar::new(b"qux")),
TextToken::Scalar(Scalar::new(b"28")),
TextToken::End(1),
]
);
let data2 = b"foo2={bar2=3 qux2=29}";
TextTape::parser()
.parse_slice_into_tape(data2, &mut tape)
.unwrap();
assert_eq!(
tape.tokens(),
vec![
TextToken::Scalar(Scalar::new(b"foo2")),
TextToken::Object(6),
TextToken::Scalar(Scalar::new(b"bar2")),
TextToken::Scalar(Scalar::new(b"3")),
TextToken::Scalar(Scalar::new(b"qux2")),
TextToken::Scalar(Scalar::new(b"29")),
TextToken::End(1),
]
);
}
#[test]
fn test_array_event() {
let data = b"versions={\r\n\t\"1.28.3.0\"\r\n}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"versions")),
TextToken::Array(3),
TextToken::Scalar(Scalar::new(b"1.28.3.0")),
TextToken::End(1),
]
);
}
#[test]
fn test_array_multievent() {
let data = b"versions={\r\n\t\"1.28.3.0\"\r\n foo}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"versions")),
TextToken::Array(4),
TextToken::Scalar(Scalar::new(b"1.28.3.0")),
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::End(1),
]
);
}
#[test]
fn test_no_equal_object_event() {
let data = b"foo{bar=qux}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Object(4),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"qux")),
TextToken::End(1),
]
);
}
#[test]
fn test_empty_array() {
let data = b"discovered_by={}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"discovered_by")),
TextToken::Array(2),
TextToken::End(1),
]
);
}
#[test]
fn test_array_of_objects() {
let data = b"stats={{id=0 type=general} {id=1 type=admiral}}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"stats")),
TextToken::Array(14),
TextToken::Object(7),
TextToken::Scalar(Scalar::new(b"id")),
TextToken::Scalar(Scalar::new(b"0")),
TextToken::Scalar(Scalar::new(b"type")),
TextToken::Scalar(Scalar::new(b"general")),
TextToken::End(2),
TextToken::Object(13),
TextToken::Scalar(Scalar::new(b"id")),
TextToken::Scalar(Scalar::new(b"1")),
TextToken::Scalar(Scalar::new(b"type")),
TextToken::Scalar(Scalar::new(b"admiral")),
TextToken::End(8),
TextToken::End(1),
]
);
}
#[test]
fn test_empty_objects() {
let data = b"history={{} 1629.11.10={core=AAA}}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"history")),
TextToken::Object(7),
TextToken::Scalar(Scalar::new(b"1629.11.10")),
TextToken::Object(6),
TextToken::Scalar(Scalar::new(b"core")),
TextToken::Scalar(Scalar::new(b"AAA")),
TextToken::End(3),
TextToken::End(1),
]
);
}
#[test]
fn test_empty_multi_objects() {
let data = b"history={{} {} 1629.11.10={core=AAA}}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"history")),
TextToken::Object(7),
TextToken::Scalar(Scalar::new(b"1629.11.10")),
TextToken::Object(6),
TextToken::Scalar(Scalar::new(b"core")),
TextToken::Scalar(Scalar::new(b"AAA")),
TextToken::End(3),
TextToken::End(1),
]
);
}
#[test]
fn test_empty_objects2() {
let data = b"foo={bar=val {}} me=you";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Object(6),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Header(Scalar::new(b"val")),
TextToken::Array(5),
TextToken::End(4),
TextToken::End(1),
TextToken::Scalar(Scalar::new(b"me")),
TextToken::Scalar(Scalar::new(b"you")),
]
);
}
#[test]
fn test_spanning_objects() {
let data = b"army={name=abc} army={name=def}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"army")),
TextToken::Object(4),
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"abc")),
TextToken::End(1),
TextToken::Scalar(Scalar::new(b"army")),
TextToken::Object(9),
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"def")),
TextToken::End(6),
]
);
}
#[test]
fn test_mixed_object_array() {
let data = br#"brittany_area = { #5
color = { 118 99 151 }
169 170 171 172 4384
}"#;
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"brittany_area")),
TextToken::Object(13),
TextToken::Scalar(Scalar::new(b"color")),
TextToken::Array(7),
TextToken::Scalar(Scalar::new(b"118")),
TextToken::Scalar(Scalar::new(b"99")),
TextToken::Scalar(Scalar::new(b"151")),
TextToken::End(3),
TextToken::Scalar(Scalar::new(b"169")),
TextToken::Scalar(Scalar::new(b"170")),
TextToken::Scalar(Scalar::new(b"171")),
TextToken::Scalar(Scalar::new(b"172")),
TextToken::Scalar(Scalar::new(b"4384")),
TextToken::End(1),
]
);
}
#[test]
fn test_regression() {
let data = [0, 32, 34, 0];
assert!(parse(&data[..]).is_err());
}
#[test]
fn test_regression2() {
let data = [0, 4, 33, 0];
let _ = parse(&data[..]);
}
#[test]
fn test_too_heavily_nested() {
let mut data = Vec::new();
data.extend_from_slice(b"foo=");
for _ in 0..100000 {
data.extend_from_slice(b"{");
}
assert!(parse(&data[..]).is_err());
}
#[test]
fn test_no_ws_comment() {
let data = b"foo=abc#def\nbar=qux";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"abc")),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"qux")),
]
);
}
#[test]
fn test_period_in_identifiers() {
let data = b"flavor_tur.8=yes";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"flavor_tur.8")),
TextToken::Scalar(Scalar::new(b"yes")),
]
);
}
#[test]
fn test_dashed_identifiers() {
let data = b"dashed-identifier=yes";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"dashed-identifier")),
TextToken::Scalar(Scalar::new(b"yes")),
]
);
}
#[test]
fn test_colon_values() {
let data = b"province_id = event_target:agenda_province";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"province_id")),
TextToken::Scalar(Scalar::new(b"event_target:agenda_province")),
]
);
}
#[test]
fn test_variables() {
let data = b"@planet_standard_scale = 11";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"@planet_standard_scale")),
TextToken::Scalar(Scalar::new(b"11")),
]
);
}
#[test]
fn test_equal_identifier() {
let data = br#"=="bar""#;
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"=")),
TextToken::Scalar(Scalar::new(b"bar")),
]
);
}
#[test]
fn test_many_line_comment() {
let mut data = Vec::new();
data.extend_from_slice(b"foo=1.000\n");
for _ in 0..100000 {
data.extend_from_slice(b"# this is a comment\n");
}
data.extend_from_slice(b"foo=2.000\n");
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"1.000")),
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"2.000")),
]
);
}
#[test]
fn test_terminating_comment() {
let data = b"# boo\r\n# baa\r\nfoo=a\r\n# bee";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Scalar(Scalar::new(b"a")),
]
);
}
#[test]
fn test_rgb_trick() {
let data = b"name = rgb ";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"rgb")),
]
);
}
#[test]
fn test_rgb_trick2() {
let data = b"name = rgb type = 4713";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"rgb")),
TextToken::Scalar(Scalar::new(b"type")),
TextToken::Scalar(Scalar::new(b"4713")),
]
);
}
#[test]
fn test_rgb_trick3() {
let data = b"name = rgbeffect";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"rgbeffect")),
]
);
}
#[test]
fn test_rgb() {
let data = b"color = rgb { 100 200 150 } ";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"color")),
TextToken::Header(Scalar::new(b"rgb")),
TextToken::Array(6),
TextToken::Scalar(Scalar::new(b"100")),
TextToken::Scalar(Scalar::new(b"200")),
TextToken::Scalar(Scalar::new(b"150")),
TextToken::End(2),
]
);
}
#[test]
fn test_hsv_trick() {
let data = b"name = hsv ";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"hsv")),
]
);
}
#[test]
fn test_hsv_trick2() {
let data = b"name = hsv type = 4713";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"hsv")),
TextToken::Scalar(Scalar::new(b"type")),
TextToken::Scalar(Scalar::new(b"4713")),
]
);
}
#[test]
fn test_hsv_trick3() {
let data = b"name = hsveffect";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"name")),
TextToken::Scalar(Scalar::new(b"hsveffect")),
]
);
}
#[test]
fn test_hsv() {
let data = b"color = hsv { 0.43 0.86 0.61 } ";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"color")),
TextToken::Header(Scalar::new(b"hsv")),
TextToken::Array(6),
TextToken::Scalar(Scalar::new(b"0.43")),
TextToken::Scalar(Scalar::new(b"0.86")),
TextToken::Scalar(Scalar::new(b"0.61")),
TextToken::End(2),
]
);
}
#[test]
fn test_hsv360() {
let data = b"color = hsv360{ 25 75 63 }";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"color")),
TextToken::Header(Scalar::new(b"hsv360")),
TextToken::Array(6),
TextToken::Scalar(Scalar::new(b"25")),
TextToken::Scalar(Scalar::new(b"75")),
TextToken::Scalar(Scalar::new(b"63")),
TextToken::End(2),
]
);
}
#[test]
fn test_cylindrical() {
let data = b"color = cylindrical{ 150 3 0 }";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"color")),
TextToken::Header(Scalar::new(b"cylindrical")),
TextToken::Array(6),
TextToken::Scalar(Scalar::new(b"150")),
TextToken::Scalar(Scalar::new(b"3")),
TextToken::Scalar(Scalar::new(b"0")),
TextToken::End(2),
]
);
}
#[test]
fn test_hex() {
let data = b"color = hex { aabbccdd }";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"color")),
TextToken::Header(Scalar::new(b"hex")),
TextToken::Array(4),
TextToken::Scalar(Scalar::new(b"aabbccdd")),
TextToken::End(2),
]
);
}
#[test]
fn test_list_header() {
let data = b"mild_winter = LIST { 3700 3701 }";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"mild_winter")),
TextToken::Header(Scalar::new(b"LIST")),
TextToken::Array(5),
TextToken::Scalar(Scalar::new(b"3700")),
TextToken::Scalar(Scalar::new(b"3701")),
TextToken::End(2),
]
);
}
#[test]
fn test_heterogenous_list() {
let data = b"levels={ 10 0=2 1=2 } foo={bar=qux}";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"levels")),
TextToken::Array(9),
TextToken::Scalar(Scalar::new(b"10")),
TextToken::HiddenObject(8),
TextToken::Scalar(Scalar::new(b"0")),
TextToken::Scalar(Scalar::new(b"2")),
TextToken::Scalar(Scalar::new(b"1")),
TextToken::Scalar(Scalar::new(b"2")),
TextToken::End(3),
TextToken::End(1),
TextToken::Scalar(Scalar::new(b"foo")),
TextToken::Object(14),
TextToken::Scalar(Scalar::new(b"bar")),
TextToken::Scalar(Scalar::new(b"qux")),
TextToken::End(11),
]
);
}
#[test]
fn test_hidden_object() {
let data = b"16778374={ levels={ 10 0=2 1=2 } }";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"16778374")),
TextToken::Object(12),
TextToken::Scalar(Scalar::new(b"levels")),
TextToken::Array(11),
TextToken::Scalar(Scalar::new(b"10")),
TextToken::HiddenObject(10),
TextToken::Scalar(Scalar::new(b"0")),
TextToken::Scalar(Scalar::new(b"2")),
TextToken::Scalar(Scalar::new(b"1")),
TextToken::Scalar(Scalar::new(b"2")),
TextToken::End(5),
TextToken::End(3),
TextToken::End(1),
]
);
}
#[test]
fn test_hidden_object_needs_key() {
let data = b"a{{}=}";
assert!(parse(&data[..]).is_err());
}
#[test]
fn test_objects_in_hidden_objects_not_supported() {
let data = b"u{1 a={0=1}";
assert!(parse(&data[..]).is_err());
}
#[test]
fn test_hidden_objects_with_headers_not_supported() {
let data = b"s{{c d=a{b=}}";
assert!(TextTape::from_slice(&data[..]).is_err());
}
#[test]
fn test_less_than_equal_operator() {
let data = b"scope:attacker.primary_title.tier <= tier_county";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"scope:attacker.primary_title.tier")),
TextToken::Operator(Operator::LessThanEqual),
TextToken::Scalar(Scalar::new(b"tier_county")),
]
);
}
#[test]
fn test_less_than_operator() {
let data = b"count < 2";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"count")),
TextToken::Operator(Operator::LessThan),
TextToken::Scalar(Scalar::new(b"2")),
]
);
}
#[test]
fn test_greater_than_operator() {
let data = b"age > 16";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"age")),
TextToken::Operator(Operator::GreaterThan),
TextToken::Scalar(Scalar::new(b"16")),
]
);
}
#[test]
fn test_greater_than_equal_operator() {
let data = b"intrigue >= high_skill_rating";
assert_eq!(
parse(&data[..]).unwrap().token_tape,
vec![
TextToken::Scalar(Scalar::new(b"intrigue")),
TextToken::Operator(Operator::GreaterThanEqual),
TextToken::Scalar(Scalar::new(b"high_skill_rating")),
]
);
}
#[test]
fn test_initial_end_does_not_panic() {
let res = parse(&b"}"[..]);
assert!(res.is_ok() || res.is_err());
}
}