panko 0.2.0

A small, zero-copy text tokenizer that crumbles strings into Words, Symbols, and Newlines.
Documentation
---
source: src/lib.rs
expression: tokens
---
[
    Newline,
    Word(
        "some",
    ),
    Symbol(
        "_",
    ),
    Word(
        "str",
    ),
    Newline,
    Symbol(
        " ",
    ),
    Newline,
    Symbol(
        " ",
    ),
    Word(
        "some",
    ),
    Symbol(
        "_",
    ),
    Word(
        "another",
    ),
    Symbol(
        "_",
    ),
    Word(
        "str",
    ),
]