Enum TokenType

Source

#[repr(u16)]pub enum TokenType {
Show 288 variants    EOF = 0,
    MacroSep = 1,
    CatchAll = 2,
    WS = 3,
    SEMI = 4,
    AMP = 5,
    PERCENT = 6,
    LPAREN = 7,
    RPAREN = 8,
    LCURLY = 9,
    RCURLY = 10,
    LBRACK = 11,
    RBRACK = 12,
    STAR = 13,
    EXCL = 14,
    EXCL2 = 15,
    BPIPE = 16,
    BPIPE2 = 17,
    PIPE2 = 18,
    STAR2 = 19,
    NOT = 20,
    FSLASH = 21,
    PLUS = 22,
    MINUS = 23,
    GTLT = 24,
    LTGT = 25,
    LT = 26,
    LE = 27,
    NE = 28,
    GT = 29,
    GE = 30,
    SoundsLike = 31,
    PIPE = 32,
    DOT = 33,
    COMMA = 34,
    COLON = 35,
    ASSIGN = 36,
    DOLLAR = 37,
    AT = 38,
    HASH = 39,
    QUESTION = 40,
    KwLT = 41,
    KwLE = 42,
    KwEQ = 43,
    KwIN = 44,
    KwNE = 45,
    KwGT = 46,
    KwGE = 47,
    KwAND = 48,
    KwOR = 49,
    KwNOT = 50,
    IntegerLiteral = 51,
    FloatLiteral = 52,
    FloatExponentLiteral = 53,
    StringLiteral = 54,
    BitTestingLiteral = 55,
    DateLiteral = 56,
    DateTimeLiteral = 57,
    NameLiteral = 58,
    TimeLiteral = 59,
    HexStringLiteral = 60,
    StringExprStart = 61,
    StringExprText = 62,
    StringExprEnd = 63,
    BitTestingLiteralExprEnd = 64,
    DateLiteralExprEnd = 65,
    DateTimeLiteralExprEnd = 66,
    NameLiteralExprEnd = 67,
    TimeLiteralExprEnd = 68,
    HexStringLiteralExprEnd = 69,
    CStyleComment = 70,
    PredictedCommentStat = 71,
    DatalinesStart = 72,
    DatalinesData = 73,
    CharFormat = 74,
    MacroComment = 75,
    MacroVarResolve = 76,
    MacroVarTerm = 77,
    MacroString = 78,
    MacroStringEmpty = 79,
    MacroLabel = 80,
    MacroIdentifier = 81,
    KwmCmpres = 82,
    KwmCompstor = 83,
    KwmDatatyp = 84,
    KwmEval = 85,
    KwmIndex = 86,
    KwmLeft = 87,
    KwmLength = 88,
    KwmLowcase = 89,
    KwmScan = 90,
    KwmSubstr = 91,
    KwmSymExist = 92,
    KwmSymGlobl = 93,
    KwmSymLocal = 94,
    KwmSysevalf = 95,
    KwmSysfunc = 96,
    KwmSysget = 97,
    KwmSysmacexec = 98,
    KwmSysmacexist = 99,
    KwmSysmexecdepth = 100,
    KwmSysmexecname = 101,
    KwmSysprod = 102,
    KwmTrim = 103,
    KwmUnquote = 104,
    KwmUpcase = 105,
    KwmVerify = 106,
    KwmKCmpres = 107,
    KwmKIndex = 108,
    KwmKLeft = 109,
    KwmKLength = 110,
    KwmKLowcase = 111,
    KwmKScan = 112,
    KwmKSubstr = 113,
    KwmKTrim = 114,
    KwmKUpcase = 115,
    KwmKVerify = 116,
    KwmValidchs = 117,
    KwmQCmpres = 118,
    KwmQLeft = 119,
    KwmQLowcase = 120,
    KwmQScan = 121,
    KwmQSubstr = 122,
    KwmQTrim = 123,
    KwmQSysfunc = 124,
    KwmQUpcase = 125,
    KwmQKCmpres = 126,
    KwmQKLeft = 127,
    KwmQKLowcase = 128,
    KwmQKScan = 129,
    KwmQKSubstr = 130,
    KwmQKTrim = 131,
    KwmQKUpcase = 132,
    KwmBquote = 133,
    KwmNrBquote = 134,
    KwmNrQuote = 135,
    KwmQuote = 136,
    KwmSuperq = 137,
    KwmStr = 138,
    KwmNrStr = 139,
    KwmAbort = 140,
    KwmCopy = 141,
    KwmDisplay = 142,
    KwmDo = 143,
    KwmTo = 144,
    KwmBy = 145,
    KwmUntil = 146,
    KwmWhile = 147,
    KwmEnd = 148,
    KwmGlobal = 149,
    KwmGoto = 150,
    KwmIf = 151,
    KwmThen = 152,
    KwmElse = 153,
    KwmInput = 154,
    KwmLet = 155,
    KwmLocal = 156,
    KwmMacro = 157,
    KwmMend = 158,
    KwmPut = 159,
    KwmReturn = 160,
    KwmSymdel = 161,
    KwmSyscall = 162,
    KwmSysexec = 163,
    KwmSyslput = 164,
    KwmSysmacdelete = 165,
    KwmSysmstoreclear = 166,
    KwmSysrput = 167,
    KwmWindow = 168,
    KwmInclude = 169,
    KwmList = 170,
    KwmRun = 171,
    Identifier = 172,
    KwEQT = 173,
    KwGTT = 174,
    KwLTT = 175,
    KwGET = 176,
    KwLET = 177,
    KwNET = 178,
    KwLibname = 179,
    KwFilename = 180,
    KwClear = 181,
    KwList = 182,
    KwCancel = 183,
    KwAllVar = 184,
    KwArray = 185,
    KwAttrib = 186,
    KwCall = 187,
    KwData = 188,
    KwDefault = 189,
    KwDescending = 190,
    KwFormat = 191,
    KwGroupformat = 192,
    KwId = 193,
    KwIf = 194,
    KwInfile = 195,
    KwInformat = 196,
    KwKeep = 197,
    KwLabel = 198,
    KwLength = 199,
    KwMerge = 200,
    KwNullDataset = 201,
    KwOutput = 202,
    KwPgm = 203,
    KwRename = 204,
    KwRun = 205,
    KwSet = 206,
    KwStop = 207,
    KwVar = 208,
    KwView = 209,
    KwWith = 210,
    KwDelete = 211,
    KwNotsorted = 212,
    KwProc = 213,
    KwQuit = 214,
    KwRanks = 215,
    KwAll = 216,
    KwAny = 217,
    KwAs = 218,
    KwAsc = 219,
    KwBetween = 220,
    KwBoth = 221,
    KwBtrim = 222,
    KwBy = 223,
    KwCalculated = 224,
    KwCase = 225,
    KwConnect = 226,
    KwConnection = 227,
    KwContains = 228,
    KwCorr = 229,
    KwCreate = 230,
    KwCross = 231,
    KwDesc = 232,
    KwDisconnect = 233,
    KwDistinct = 234,
    KwDo = 235,
    KwDrop = 236,
    KwElse = 237,
    KwEnd = 238,
    KwEscape = 239,
    KwExcept = 240,
    KwExecute = 241,
    KwExists = 242,
    KwFor = 243,
    KwFrom = 244,
    KwFull = 245,
    KwGroup = 246,
    KwHaving = 247,
    KwIndex = 248,
    KwInner = 249,
    KwInsert = 250,
    KwIntersect = 251,
    KwInto = 252,
    KwIs = 253,
    KwJoin = 254,
    KwKey = 255,
    KwLeading = 256,
    KwLeft = 257,
    KwLike = 258,
    KwMissing = 259,
    KwNatural = 260,
    KwNotrim = 261,
    KwNull = 262,
    KwOn = 263,
    KwOrder = 264,
    KwOuter = 265,
    KwPrimary = 266,
    KwRight = 267,
    KwSelect = 268,
    KwSeparated = 269,
    KwSubstring = 270,
    KwTable = 271,
    KwThen = 272,
    KwTo = 273,
    KwTrailing = 274,
    KwTrimmed = 275,
    KwUnion = 276,
    KwUnique = 277,
    KwUpdate = 278,
    KwUsing = 279,
    KwValues = 280,
    KwWhen = 281,
    KwWhere = 282,
    KwDeclare = 283,
    KwHash = 284,
    KwHiter = 285,
    KwInput = 286,
    KwPut = 287,
}

Expand description

What you expect - the token types.

Order of variants are VERY important, as multiple predicates rely on integer values of the variants that are generated automatically from the order.

Naming is also important. Namely all non-macro keywords must be preceded with Kw and macro keywords are all start with Kwm. These are used to autogenerate static hash maps for keywords and other downstream codegen outside of this crate

Variants§

§

EOF = 0

§

MacroSep = 1

A special virtual token that is emitted between open code and macro statements when there is no “natural” separator, or when semicolon is missing between two macro statements (a coding error).

This may be used by a downstream parser as a reliable terminating token for dynamic open code and thus avoid doing lookaheads. Dynamic, means that the statement has a macro statements in it, like data %if cond %then %do; t1 %end; %else %do; t2 %end;;

It is emitted when macro_sep feature is enabled.

§