Struct jomini::binary::Lexer

pub struct Lexer<'a> { /* private fields */ }

Zero cost binary data scanner.

There are two main ways to drive the lexer. To see them in action, imagine we want to count the max amount of nesting.

use jomini::binary::{Lexer, Token};
let mut lexer = Lexer::new(&[0x2d, 0x28, 0x01, 0x00, 0x03, 0x00, 0x03, 0x00, 0x04, 0x00, 0x04, 0x00]);
let mut max_depth = 0;
let mut current_depth = 0;
while let Some(token) = lexer.next_token()? {
  match token {
    Token::Open => {
      current_depth += 1;
      max_depth = max_depth.max(current_depth);
    }
    Token::Close => current_depth -= 1,
    _ => {}
  }
}
assert_eq!(max_depth, 2);

The Lexer::next_token is an ergonomic way to scan through binary tokens. The functions prefixed with read_denote more data is expected, while next_ allows for the data to finish.

If it is desired scan through the binary data with zero overhead, one needs to drive the lexer more thoroughly.

use jomini::binary::{Lexer, LexemeId};
let mut lexer = Lexer::new(&[0x2d, 0x28, 0x01, 0x00, 0x03, 0x00, 0x03, 0x00, 0x04, 0x00, 0x04, 0x00]);
let mut max_depth = 0;
let mut current_depth = 0;
while let Some(id) = lexer.next_id()? {
  match id {
    LexemeId::OPEN => {
      current_depth += 1;
      max_depth = max_depth.max(current_depth);
    }
    LexemeId::CLOSE => current_depth -= 1,
    LexemeId::U32 => { lexer.read_u32()?; }
    LexemeId::I32 => { lexer.read_i32()?; }
    LexemeId::BOOL => { lexer.read_bool()?; }
    LexemeId::QUOTED | LexemeId::UNQUOTED => { lexer.read_string()?; }
    LexemeId::F32 => { lexer.read_f32()?; }
    LexemeId::F64 => { lexer.read_f64()?; }
    LexemeId::I64 => { lexer.read_i64()?; }
    _ => {}
  }
}
assert_eq!(max_depth, 2);

Only at token boundaries can token functions be intertwined with the individual lexeme functions.

Errors reported will contain positional information.

Implementations

impl<'a> Lexer<'a>

pub fn new(data: &'a [u8]) -> Self

Creates a new lexer over the given data

pub fn remainder(&self) -> &'a [u8]

Returns the remaining data that has not yet been processed.

use jomini::binary::{Lexer, LexemeId};
let mut lexer = Lexer::new(&[0xd2, 0x28, 0xff]);
assert_eq!(lexer.read_id().unwrap(), LexemeId::new(0x28d2));
assert_eq!(lexer.remainder(), &[0xff]);

pub fn position(&self) -> usize

Returns how many bytes have been processed by the lexer

use jomini::binary::{Lexer, LexemeId};
let mut lexer = Lexer::new(&[0xd2, 0x28, 0xff]);
assert_eq!(lexer.read_id().unwrap(), LexemeId::new(0x28d2));
assert_eq!(lexer.position(), 2);

pub fn read_id(&mut self) -> Result<LexemeId, LexerError>

Advance the lexer through the next lexeme id, and return it

use jomini::binary::{Lexer, LexemeId, LexError};
let mut lexer = Lexer::new(&[0x2d, 0x28]);
assert_eq!(lexer.read_id(), Ok(LexemeId::new(0x282d)));
assert_eq!(lexer.read_id().unwrap_err().kind(), &LexError::Eof);

pub fn next_id(&mut self) -> Result<Option<LexemeId>, LexerError>

Attempt to advance through the LexemeId

An EOF error can still be thrown if data is present but not enough exists to decode the next LexemeId

use jomini::binary::{Lexer, LexemeId, LexError};
let mut lexer = Lexer::new(&[0x2d, 0x28]);
assert_eq!(lexer.next_id(), Ok(Some(LexemeId::new(0x282d))));
assert_eq!(lexer.next_id(), Ok(None));

let mut lexer = Lexer::new(&[0x2d]);
assert_eq!(lexer.next_id().unwrap_err().kind(), &LexError::Eof);

pub fn read_token(&mut self) -> Result<Token<'a>, LexerError>

Assume more tokens exist in the data and read the next one.

use jomini::binary::{Lexer, LexError, Token};
let mut lexer = Lexer::new(&[0x2d, 0x28]);
assert_eq!(lexer.read_token(), Ok(Token::Id(0x282d)));
assert_eq!(lexer.read_token().unwrap_err().kind(), &LexError::Eof);

pub fn next_token(&mut self) -> Result<Option<Token<'a>>, LexerError>

Attempt to advance through the next token or return None if no data remains

An EOF error can still be thrown if data is present but not enough exists to decode the next token.

use jomini::binary::{Lexer, Token, LexError};
let mut lexer = Lexer::new(&[0x2d, 0x28]);
assert_eq!(lexer.next_token(), Ok(Some(Token::Id(0x282d))));
assert_eq!(lexer.next_token(), Ok(None));

let mut lexer = Lexer::new(&[0x2d]);
assert_eq!(lexer.next_token().unwrap_err().kind(), &LexError::Eof);

pub fn peek_id(&mut self) -> Option<LexemeId>

Peek at the next LexemeId without advancing the lexer

use jomini::binary::{Lexer, LexError, LexemeId};
let mut lexer = Lexer::new(&[0x01, 0x00][..]);
assert_eq!(lexer.peek_id(), Some(LexemeId::EQUAL));
assert_eq!(lexer.read_id(), Ok(LexemeId::EQUAL));
assert_eq!(lexer.peek_id(), None);

pub fn peek_token(&mut self) -> Option<Token<'a>>

Peek at the next Token without advancing the lexer

use jomini::binary::{Lexer, LexError, Token};
let mut lexer = Lexer::new(&[0x01, 0x00][..]);
assert_eq!(lexer.peek_token(), Some(Token::Equal));
assert_eq!(lexer.read_token(), Ok(Token::Equal));
assert_eq!(lexer.peek_token(), None);

pub fn read_string(&mut self) -> Result<Scalar<'a>, LexerError>

Advance the lexer through a length prefixed string

use jomini::{Scalar, binary::{Lexer, LexError}};
let mut lexer = Lexer::new(&[0x03, 0x00, 0x45, 0x4e, 0x47][..]);
assert_eq!(lexer.read_string(), Ok(Scalar::new(b"ENG")));
assert_eq!(lexer.read_string().unwrap_err().kind(), &LexError::Eof);

pub fn read_bool(&mut self) -> Result<bool, LexerError>

Advance the lexer through a boolean

use jomini::binary::{Lexer, LexError};
let mut lexer = Lexer::new(&[0x01, 0x00][..]);
assert_eq!(lexer.read_bool(), Ok(true));
assert_eq!(lexer.read_bool(), Ok(false));
assert_eq!(lexer.read_bool().unwrap_err().kind(), &LexError::Eof);

pub fn read_u32(&mut self) -> Result<u32, LexerError>

Advance the lexer through unsigned little endian 32 bit integer

use jomini::binary::{Lexer, LexError};
let mut lexer = Lexer::new(&[0x59, 0x00, 0x00, 0x00][..]);
assert_eq!(lexer.read_u32(), Ok(89));
assert_eq!(lexer.read_u32().unwrap_err().kind(), &LexError::Eof);

pub fn read_u64(&mut self) -> Result<u64, LexerError>

Advance the lexer through unsigned little endian 64 bit integer

use jomini::binary::{Lexer, LexError};
let mut lexer = Lexer::new(&[0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]);
assert_eq!(lexer.read_u64(), Ok(128));
assert_eq!(lexer.read_u64().unwrap_err().kind(), &LexError::Eof);

pub fn read_i64(&mut self) -> Result<i64, LexerError>

Advance the lexer through signed little endian 64 bit integer

use jomini::binary::{Lexer, LexError};
let mut lexer = Lexer::new(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff][..]);
assert_eq!(lexer.read_i64(), Ok(-1));
assert_eq!(lexer.read_i64().unwrap_err().kind(), &LexError::Eof);

pub fn read_i32(&mut self) -> Result<i32, LexerError>

Advance the lexer through signed little endian 32 bit integer

use jomini::binary::{Lexer, LexError};
let mut lexer = Lexer::new(&[0x59, 0x00, 0x00, 0x00][..]);
assert_eq!(lexer.read_i32(), Ok(89));
assert_eq!(lexer.read_i32().unwrap_err().kind(), &LexError::Eof);

pub fn read_f32(&mut self) -> Result<[u8; 4], LexerError>

Advance the lexer through 32 bits of floating point data and return the bytes

use jomini::binary::{Lexer, LexError};
let data = [0x17, 0x00, 0x00, 0x00];
let mut lexer = Lexer::new(&data[..]);
assert_eq!(lexer.read_f32(), Ok(data));
assert_eq!(lexer.read_f32().unwrap_err().kind(), &LexError::Eof);

pub fn read_f64(&mut self) -> Result<[u8; 8], LexerError>

Advance the lexer through 64 bits of floating point data and return the bytes

use jomini::binary::{Lexer, LexError};
let data = [0xc7, 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
let mut lexer = Lexer::new(&data[..]);
assert_eq!(lexer.read_f64(), Ok(data));
assert_eq!(lexer.read_f64().unwrap_err().kind(), &LexError::Eof);

pub fn read_rgb(&mut self) -> Result<Rgb, LexerError>

Advance the lexer through an rgb value (with optional alpha channel)

use jomini::binary::{Lexer, LexError, Rgb};
let data = [0x03, 0x00, 0x14, 0x00, 0x6e, 0x00, 0x00, 0x00,
            0x14, 0x00, 0x1b, 0x00, 0x00, 0x00, 0x14, 0x00,
            0x1b, 0x00, 0x00, 0x00, 0x04, 0x00];
let mut lexer = Lexer::new(&data[..]);
assert_eq!(lexer.read_rgb(), Ok(Rgb { r: 110, g: 27, b: 27, a: None }));
assert_eq!(lexer.read_rgb().unwrap_err().kind(), &LexError::Eof);

pub fn read_bytes(&mut self, bytes: usize) -> Result<&'a [u8], LexerError>

Advance a given number of bytes and return them

use jomini::binary::{Lexer, LexError};
let mut lexer = Lexer::new(b"EU4bin");
assert_eq!(lexer.read_bytes(6), Ok(&b"EU4bin"[..]));
assert_eq!(lexer.read_bytes(1).unwrap_err().kind(), &LexError::Eof);

pub fn skip_value(&mut self, id: LexemeId) -> Result<(), LexerError>

Skip the value denoted by the LexemeId. Will skip entire containers.