ocomment_core/lexical_pool.rs
1//! The alphabet the randomised property tests draw their sources from.
2//!
3//! Two suites generate sources this way — the checkpoint and incremental
4//! properties in `src/incremental.rs`, and the whole-file properties in
5//! `tests/properties.rs` — and they are meant to draw from the same alphabet: a
6//! fragment worth generating against the whole-file scanner is worth generating
7//! against the incremental one, because the incremental engine's promise is
8//! that a restart reproduces what the whole-file scan would have said. One is a
9//! unit test inside the crate and the other an integration test outside it, and
10//! the only thing both can name is the crate's public surface, so the alphabet
11//! lives here rather than being written out twice.
12//!
13//! It is `#[doc(hidden)]` and carries no stability promise: it is test support
14//! that happens to have to be reachable from outside.
15
16/// Single bytes that reach every built-in scanner's string, comment, here
17/// document and template states rather than only the C-family delimiters.
18///
19/// A generator draws one of these against a much smaller weight of uniform
20/// random bytes, so a delimiter arrives often enough for two of them to meet.
21/// Each byte appears once and is drawn as often as the next; a caller that
22/// wants one of them oftener says so with a weight of its own.
23#[doc(hidden)]
24pub const BYTES: &[u8] = b"\n\r/*'\"#`{}<>=[]-|?\\$%()@:!~";
25
26/// Multi-byte tokens a single-byte alphabet can never synthesise.
27///
28/// The preamble and directive rules only fire on whole words, so without these
29/// the generated sources never reach the code paths that make a scan depend on
30/// where in the document it starts.
31///
32/// The two triple-quote runs are here for the opposite reason: they are three
33/// of one byte, which a per-byte alphabet reaches only by coincidence, and they
34/// open a string that swallows newlines in Python, Kotlin, Java, and TOML —
35/// which is exactly the state a restart must not be allowed to land inside.
36/// Lua's long brackets are the same state behind four bytes rather than three,
37/// and the levelled forms are here because a closing bracket of the wrong level
38/// is content: without them a generated source that opens one practically never
39/// closes it. The eight YAML fragments after them are block scalar headers and
40/// the indented line that follows one — a body is the state a YAML restart must
41/// never land inside, and the bytes that open one have to arrive in that order.
42/// Three of the eight put the owner of that body on an earlier line than its
43/// header: a line ending in `:` or in a bare `-`, and the node properties that
44/// may stand between the two. That owner is the one thing a YAML line does not
45/// say about itself, so it is the one thing a restart at a line start has to be
46/// refused over. The `|+` header is the keep-chomped body whose trailing blank
47/// lines are content. The five PHP fragments after those are its two tags, an
48/// attribute, and a here document header with the line that closes one: PHP
49/// mode is the state a restart must never land inside, and only a whole `<?php`
50/// opens it. The Ruby fragments at the end are its two column-zero markers with
51/// the line each of them needs, a here document header with the terminator that
52/// ends one, a percent literal opener, and the interpolation boundary a here
53/// document header may be written across: four of Ruby's tokens are spelled
54/// with a byte that is also an operator, so only the whole opener reaches the
55/// state, and an embedded document, a here document body and the DATA section
56/// are three more states a restart must never land inside. The body a header
57/// inside `"#{ ... }"` asks for belongs to the line the header stands on rather
58/// than to the interpolation, and only a pool that can assemble that opener
59/// generates the case at all. The two Zig fragments at the end are the whole
60/// opener of a multiline string literal line — two of one byte, which a
61/// per-byte alphabet reaches only by coincidence, and the only thing in the
62/// language that hides a `//` without a quote — and the fourth slash that turns
63/// a documentation comment back into an ordinary one. The five R fragments
64/// after them are its raw string openers and closers and its roxygen marker:
65/// the `r` that opens a raw string is a letter no per-byte alphabet carries, so
66/// without them a generated R source never reaches the one literal in the
67/// language whose delimiter is neither a quote nor a bracket alone, and the
68/// dashed pair is there for the reason Lua's levelled brackets are — a closing
69/// run of the wrong length is content. The three Dart fragments after those are
70/// its raw string openers, whose `r` is a letter no per-byte alphabet carries
71/// and which take neither an escape nor an interpolation, and the interpolation
72/// opener itself: a `${` turns the inside of a string back into code, and a
73/// comment written there is one more state a restart must not land inside. The
74/// five Swift fragments after them are its raw-string and extended-regex
75/// delimiters with the escape the hashes rename: `#"` and `"#` open and close
76/// the raw string whose escape is `\#(`, and `#/` and `/#` the regular
77/// expression literal that may hold an unescaped `/` and may span lines. Each
78/// is two or more bytes that have to arrive in that order, and each opens a
79/// state a restart must not land inside. The seven C# fragments at the end are
80/// its four string openers, the brace pair that is an escape in one form and a
81/// hole in another, and two directive words: an `@` and a quote and a `$@` and
82/// a quote carry line breaks, a `$` and a quote carries a hole that may, a run
83/// of two `$` in front of three quotes needs two braces to open one, and a line
84/// whose first non-blank byte opens a directive is lexed by rules of its own.
85/// Each is a state a restart must not land inside, and none of them is a shape
86/// a per-byte alphabet assembles.
87#[doc(hidden)]
88pub const TOKENS: &[&[u8]] = &[
89 b"coding:",
90 b"# -*- coding: utf-8 -*-",
91 b"# coding: latin-1",
92 b"#!",
93 b"//go:build",
94 b"/*#__PURE__*/",
95 b"<!--",
96 b"r#\"",
97 b"\"\"\"",
98 b"'''",
99 b"--[[",
100 b"--[=[",
101 b"]]",
102 b"]=]",
103 b": |\n",
104 b"- >2\n",
105 b"|+\n",
106 b"\n # ",
107 b"k:\n",
108 b"\n-\n",
109 b"!!str ",
110 b"&a ",
111 b"<?php ",
112 b"<?=",
113 b"#[",
114 b"<<<E\n",
115 b"\nE;\n",
116 b"=begin ",
117 b"\n=end\n",
118 b"__END__\n",
119 b" <<~EOS\n",
120 b"\nEOS\n",
121 b"%w[",
122 b"\"#{",
123 b"}\"",
124 b"\"#{ <<EOS }\"",
125 b"\\\\",
126 b"////",
127 b"r\"(",
128 b")\"",
129 b"r\"--(",
130 b")--\"",
131 b"#'",
132 b"r'",
133 b"r'''",
134 b"${",
135 b"#\"",
136 b"\"#",
137 b"\\#(",
138 b"#/",
139 b"/#",
140 b"@\"",
141 b"$\"",
142 b"$@\"",
143 b"$$\"\"\"",
144 b"{{",
145 b"#if ",
146 b"#region ",
147 b"\"\"\"\"",
148 b"$${",
149 b"'\"'",
150 b"#{",
151 b"url(",
152 b"<template data-lang=",
153 b"v-pre",
154 b"on:click={",
155 b"<style lang=\"sass\">",
156 b"```",
157 b"{r, ",
158 b" <!--",
159 b"$#",
160 b"__DATA__\n",
161 b"=pod\n",
162 b"=cut\n",
163 b"s///",
164 b"tr///",
165];