Module tokenizer_lib::StreamedTokenChannel

Structs

StreamedTokenReader
StreamedTokenSender

Functions

new	Will return a TokenSender and TokenReader for use when lexing and parsing in separate threads Unlike StaticTokenChannel it does not buffer all the tokens before parsing can begin