Struct castle_tokenizer::Tokenizer

source · [−]

pub struct Tokenizer<R> {
    pub cursor: Cursor<R>,
    pub peeked: VecDeque<Token>,
}

Fields

cursor: Cursor<R>peeked: VecDeque<Token>

Implementations

source

impl<R: Read> Tokenizer<R>

source

pub fn new(reader: R) -> Self

source

pub fn advance(&mut self) -> Result<Option<Token>, CastleError>

Advances the cursor and returns the next token Skips comments and whitespace (not including line terminators) Coalesces consecutive line terminators (\n and \r)

Trait Implementations

source

impl<R: Read> Tokenizable for Tokenizer<R>

source

fn next(
&mut self,
skip_line_terminators: bool
) -> Result<Option<Token>, CastleError>

source

fn peek_n(
 &mut self,
 skip_n: usize,
 skip_line_terminators: bool
) -> Result<Option<&Token>, CastleError>

source

fn peek(
&mut self,
skip_line_terminators: bool
) -> Result<Option<&Token>, CastleError>

source

fn peek_expect(
&mut self,
skip_line_terminators: bool
) -> Result<&Token, CastleError>

source

fn peek_token_kind(
&mut self,
skip_line_terminators: bool
) -> Result<Option<&TokenKind>, CastleError>

Auto Trait Implementations

impl<R> RefUnwindSafe for Tokenizer<R> where
R: RefUnwindSafe,

impl<R> Send for Tokenizer<R> where
R: Send,

impl<R> Sync for Tokenizer<R> where
R: Sync,

impl<R> Unpin for Tokenizer<R> where
R: Unpin,

impl<R> UnwindSafe for Tokenizer<R> where
R: UnwindSafe,

Blanket Implementations

source

impl<T> Any for T where
T: 'static + ?Sized,

source

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

source

impl<T> Borrow<T> for T where
T: ?Sized,

const: unstable · source

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

source

impl<T> BorrowMut<T> for T where
T: ?Sized,

const: unstable · source

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

source

impl<T> ExpectIdentifier for T where
T: Tokenizable,

source

fn expect_identifier(
&mut self,
skip_line_terminators: bool
) -> Result<Box<str>, CastleError>

source

impl<T> ExpectKeyword for T where
T: Tokenizable,

source

fn expect_keyword(
 &mut self,
 keyword: Keyword,
 skip_line_terminators: bool
) -> Result<(), CastleError>

source

impl<T> ExpectNext for T where
T: Tokenizable,

source

fn expect_next(
&mut self,
skip_line_terminators: bool
) -> Result<Token, CastleError>

source

impl<T> ExpectPrimitive for T where
T: Tokenizable,

source

fn expect_primitive(
&mut self,
skip_line_terminators: bool
) -> Result<Primitive, CastleError>

source

impl<T> ExpectPunctuator for T where
T: Tokenizable,

source

fn expect_punctuator(
 &mut self,
 punctuator: Punctuator,
 skip_line_terminators: bool
) -> Result<(), CastleError>

source

impl<T> From<T> for T

const: unstable · source

fn from(t: T) -> T

Returns the argument unchanged.

source

impl<T, U> Into for T where
U: From<T>,

const: unstable · source

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

source

impl<T> IsPunctuator for T where
T: Tokenizable,

source

fn peek_is_punctuator(
 &mut self,
 expected: Punctuator,
 skip_line_terminators: bool
) -> Result<bool, CastleError>

source

impl<T> PeekKeyword for T where
T: Tokenizable,

source

fn peek_keyword(
&mut self,
skip_line_terminators: bool
) -> Result<Option<&Keyword>, CastleError>

source

impl<T, U> TryFrom for T where
U: Into<T>,

type Error = Infallible

The type returned in the event of a conversion error.

const: unstable · source

fn try_from(value: U) -> Result<T, <T as TryFrom>::Error>

Performs the conversion.

source

impl<T, U> TryInto for T where
U: TryFrom<T>,

type Error = >::Error

The type returned in the event of a conversion error.

const: unstable · source

fn try_into(self) -> Result<U, >::Error>

Performs the conversion.

Struct castle_tokenizer::Tokenizer

Fields

Implementations

impl<R: Read> Tokenizer<R>

pub fn new(reader: R) -> Self

pub fn advance(&mut self) -> Result<Option<Token>, CastleError>

Trait Implementations

impl<R: Read> Tokenizable for Tokenizer<R>

fn next( &mut self, skip_line_terminators: bool) -> Result<Option<Token>, CastleError>

fn peek_n( &mut self, skip_n: usize, skip_line_terminators: bool) -> Result<Option<&Token>, CastleError>

fn peek( &mut self, skip_line_terminators: bool) -> Result<Option<&Token>, CastleError>

fn peek_expect( &mut self, skip_line_terminators: bool) -> Result<&Token, CastleError>

fn peek_token_kind( &mut self, skip_line_terminators: bool) -> Result<Option<&TokenKind>, CastleError>

Auto Trait Implementations

impl<R> RefUnwindSafe for Tokenizer<R> where R: RefUnwindSafe,

impl<R> Send for Tokenizer<R> where R: Send,

impl<R> Sync for Tokenizer<R> where R: Sync,

impl<R> Unpin for Tokenizer<R> where R: Unpin,

impl<R> UnwindSafe for Tokenizer<R> where R: UnwindSafe,

Blanket Implementations

impl<T> Any for T where T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for T where T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for T where T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> ExpectIdentifier for T where T: Tokenizable,

fn expect_identifier( &mut self, skip_line_terminators: bool) -> Result<Box<str>, CastleError>

impl<T> ExpectKeyword for T where T: Tokenizable,

fn expect_keyword( &mut self, keyword: Keyword, skip_line_terminators: bool) -> Result<(), CastleError>

impl<T> ExpectNext for T where T: Tokenizable,

fn expect_next( &mut self, skip_line_terminators: bool) -> Result<Token, CastleError>

impl<T> ExpectPrimitive for T where T: Tokenizable,

fn expect_primitive( &mut self, skip_line_terminators: bool) -> Result<Primitive, CastleError>

impl<T> ExpectPunctuator for T where T: Tokenizable,

fn expect_punctuator( &mut self, punctuator: Punctuator, skip_line_terminators: bool) -> Result<(), CastleError>

impl<T> From<T> for T

fn from(t: T) -> T

impl<T, U> Into<U> for T where U: From<T>,

fn into(self) -> U

impl<T> IsPunctuator for T where T: Tokenizable,

fn peek_is_punctuator( &mut self, expected: Punctuator, skip_line_terminators: bool) -> Result<bool, CastleError>

impl<T> PeekKeyword for T where T: Tokenizable,

fn peek_keyword( &mut self, skip_line_terminators: bool) -> Result<Option<&Keyword>, CastleError>

impl<T, U> TryFrom<U> for T where U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for T where U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

fn next(
&mut self,
skip_line_terminators: bool
) -> Result<Option<Token>, CastleError>

fn peek_n(
&mut self,
skip_n: usize,
skip_line_terminators: bool
) -> Result<Option<&Token>, CastleError>

fn peek(
&mut self,
skip_line_terminators: bool
) -> Result<Option<&Token>, CastleError>

fn peek_expect(
&mut self,
skip_line_terminators: bool
) -> Result<&Token, CastleError>

fn peek_token_kind(
&mut self,
skip_line_terminators: bool
) -> Result<Option<&TokenKind>, CastleError>

impl<R> RefUnwindSafe for Tokenizer<R> where
R: RefUnwindSafe,

impl<R> Send for Tokenizer<R> where
R: Send,

impl<R> Sync for Tokenizer<R> where
R: Sync,

impl<R> Unpin for Tokenizer<R> where
R: Unpin,

impl<R> UnwindSafe for Tokenizer<R> where
R: UnwindSafe,

impl<T> Any for T where
T: 'static + ?Sized,

impl<T> Borrow<T> for T where
T: ?Sized,

impl<T> BorrowMut<T> for T where
T: ?Sized,

impl<T> ExpectIdentifier for T where
T: Tokenizable,

fn expect_identifier(
&mut self,
skip_line_terminators: bool
) -> Result<Box<str>, CastleError>

impl<T> ExpectKeyword for T where
T: Tokenizable,

fn expect_keyword(
&mut self,
keyword: Keyword,
skip_line_terminators: bool
) -> Result<(), CastleError>

impl<T> ExpectNext for T where
T: Tokenizable,

fn expect_next(
&mut self,
skip_line_terminators: bool
) -> Result<Token, CastleError>

impl<T> ExpectPrimitive for T where
T: Tokenizable,

fn expect_primitive(
&mut self,
skip_line_terminators: bool
) -> Result<Primitive, CastleError>

impl<T> ExpectPunctuator for T where
T: Tokenizable,

fn expect_punctuator(
&mut self,
punctuator: Punctuator,
skip_line_terminators: bool
) -> Result<(), CastleError>

impl<T, U> Into<U> for T where
U: From<T>,

impl<T> IsPunctuator for T where
T: Tokenizable,

fn peek_is_punctuator(
&mut self,
expected: Punctuator,
skip_line_terminators: bool
) -> Result<bool, CastleError>

impl<T> PeekKeyword for T where
T: Tokenizable,

fn peek_keyword(
&mut self,
skip_line_terminators: bool
) -> Result<Option<&Keyword>, CastleError>

impl<T, U> TryFrom<U> for T where
U: Into<T>,

impl<T, U> TryInto<U> for T where
U: TryFrom<T>,