use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec;
use alloc::vec::Vec;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenKind {
Name,
Keyword,
Number,
String,
UnclosedText,
Op,
Directive,
Newline,
Eof,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub value: String,
pub line: u32,
pub column: u32,
pub spaced: bool,
pub quote: Option<char>,
}
pub const KEYWORDS: &[&str] = &[
"if", "else", "switch", "case", "return", "exitapp", "while", "until", "loop", "try", "catch",
"finally", "throw", "for", "break", "continue", "goto", "true", "false", "and", "or", "not",
"is", "in", "contains", "as", "class", "static", "global", "local", "super",
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TheRefusal {
Variable,
Syntax,
Future,
Func,
Class,
}
fn the_end_of_a_symbol_that_carries_a_quote(chars: &[char], start: usize) -> usize {
let mut end = start;
while end < chars.len() && !the_character_ends_a_symbol(chars[end]) {
end += 1;
}
end
}
fn the_character_ends_a_symbol(character: char) -> bool {
character.is_whitespace()
|| matches!(
character,
',' | ')'
| '}'
| '{'
| ':'
| '.'
| ';'
| '='
| '!'
| '<'
| '>'
| '&'
| '|'
| '^'
| '+'
| '-'
| '*'
| '/'
| '%'
| '~'
| '?'
| '('
| '['
)
}
pub fn the_character_ends_the_symbol_of_the_v1(character: char) -> bool {
character.is_whitespace()
|| matches!(
character,
',' | ')'
| '('
| '}'
| '{'
| ']'
| '['
| ':'
| '='
| '!'
| '<'
| '>'
| '&'
| '|'
| '^'
| '+'
| '-'
| '*'
| '/'
| '~'
| '?'
)
}
fn the_source_of_a_text(chars: &[char], start: usize) -> String {
let mut end = start + 1;
while end < chars.len() && chars[end] != '"' && chars[end] != '\n' {
end += 1;
}
if end < chars.len() && chars[end] == '"' {
end += 1;
}
chars[start..end].iter().collect()
}
pub fn the_refusal_of_a_word(word: &str) -> Option<TheRefusal> {
let word = word.to_lowercase();
Some(match word.as_str() {
"if" | "else" | "switch" | "case" | "return" | "while" | "until" | "loop" | "try"
| "catch" | "finally" | "throw" | "for" | "break" | "continue" | "goto" | "static"
| "global" | "local" => TheRefusal::Variable,
"not" | "and" | "or" | "is" | "true" | "false" | "super" => TheRefusal::Syntax,
"in" | "contains" | "as" => TheRefusal::Future,
"exitapp" => TheRefusal::Func,
"class" => TheRefusal::Class,
_ => return None,
})
}
const OPERATORS: &[&str] = &[
">>>=", "<<=", "//=", ">>=", ":=", "+=", "-=", "*=", "/=", ".=", "|=", "^=", "&=", "~=", "!==",
"==", "!=", "<=", ">=", "&&", "||", "??", "**", "//", "<<", ">>>", ">>", "++", "--", "::",
"=>", "=", "<", ">", "+", "-", "*", "%", "/", ".", "(", ")", ",", "{", "}", "[", "]", "!", ":",
"?", "&", "|", "^", "~",
];
fn the_hotstring_of_the_v2_has_a_block(text: &str) -> bool {
let text = text.trim_end();
text.ends_with('{') || text.rsplit("::").next().map(str::is_empty).unwrap_or(false)
}
fn the_block_of_the_hotstring_follows(chars: &[char], text: &str, index: usize) -> bool {
let text = text.trim_end();
if text.ends_with('{') {
return true;
}
let mut at = index;
if at < chars.len() && chars[at] == '\r' {
at += 1;
}
if at < chars.len() && chars[at] == '\n' {
at += 1;
}
while at < chars.len() && (chars[at] == ' ' || chars[at] == '\t' || chars[at] == '\r') {
at += 1;
}
at < chars.len() && chars[at] == '{'
}
fn the_rest_of_the_line_is_blank(chars: &[char], from: usize) -> bool {
let mut at = from;
while at < chars.len() && chars[at] != '\n' {
if chars[at] != ' ' && chars[at] != '\t' && chars[at] != '\r' {
return false;
}
at += 1;
}
true
}
fn the_line_from_has_the_two_colons(chars: &[char], from: usize) -> bool {
let mut at = from;
while at < chars.len() && chars[at] != '\n' {
if chars[at] == ':' && chars.get(at + 1) == Some(&':') {
return true;
}
at += 1;
}
false
}
fn the_operators_of(dialect: &'static dyn crate::dialect::Dialect) -> Vec<&'static str> {
if dialect.the_lexer_knows_the_null_assignment() {
let mut operators = OPERATORS.to_vec();
let at = operators
.iter()
.position(|operator| *operator == "??")
.unwrap_or(0);
operators.insert(at, "??=");
return operators;
}
if !dialect.the_lexer_knows_the_legacy_operators() {
return OPERATORS.to_vec();
}
let mut operators = OPERATORS.to_vec();
let at = operators
.iter()
.position(|operator| *operator == "<")
.unwrap_or(0);
operators.insert(at, "<>");
operators
}
#[derive(Clone, Debug)]
pub struct SyntaxError {
pub message: String,
pub line: u32,
pub column: u32,
}
impl core::fmt::Display for SyntaxError {
fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(
formatter,
"{} (line {}, column {})",
self.message, self.line, self.column
)
}
}
#[cfg(feature = "std")]
impl std::error::Error for SyntaxError {}
impl SyntaxError {
fn new(message: impl Into<String>, line: u32, column: u32) -> Self {
Self {
message: message.into(),
line,
column,
}
}
}
const THE_DIRECTIVES: [&str; 19] = [
"clipboardtimeout",
"dllload",
"errorstdout",
"hotif",
"hotiftimeout",
"hotstring",
"include",
"includeagain",
"inputlevel",
"maxthreads",
"maxthreadsbuffer",
"maxthreadsperhotkey",
"notrayicon",
"requires",
"singleinstance",
"suspendexempt",
"usehook",
"warn",
"winactivateforce",
];
pub fn carries_directive(name: &str) -> bool {
THE_DIRECTIVES.contains(&name.to_lowercase().as_str())
}
fn the_character_is_a_letter(character: char, marks: bool) -> bool {
character.is_alphabetic()
|| character == '_'
|| character == '\u{feff}'
|| (marks && matches!(character, '#' | '$' | '@'))
}
fn the_character_is_of_a_name(character: char, marks: bool) -> bool {
character.is_alphanumeric()
|| character == '_'
|| character == '\u{feff}'
|| (marks && matches!(character, '#' | '$' | '@'))
}
struct TheNameWithTheEscape {
name: String,
end: usize,
operator: Option<String>,
has_an_escape: bool,
}
fn the_name_with_the_escape(
chars: &[char],
start: usize,
marks: bool,
dialect: &'static dyn crate::dialect::Dialect,
) -> TheNameWithTheEscape {
let mut name = String::new();
let mut index = start;
let mut operator = None;
let mut has_an_escape = false;
while index < chars.len() {
let character = chars[index];
if the_character_is_of_a_name(character, marks) {
name.push(character);
index += 1;
continue;
}
if character != '`' || !dialect.the_escape_may_be_in_a_name() {
break;
}
has_an_escape = true;
let Some(next) = chars.get(index + 1).copied() else {
index += 1;
break;
};
index += 2;
let expanded = dialect.the_escape_of_the_dialect(next);
if expanded.is_whitespace() {
break;
}
if expanded == ',' {
break;
}
if OPERATORS.contains(&expanded.to_string().as_str()) {
operator = Some(expanded.to_string());
break;
}
name.push(expanded);
}
TheNameWithTheEscape {
name,
end: index,
operator,
has_an_escape,
}
}
fn the_name_of_a_message(name: &str) -> String {
format!("{:?}", name.replace('\u{feff}', "?"))
}
fn the_line_of_the_hash(chars: &[char], start: usize) -> String {
let mut end = start;
while end < chars.len() && chars[end] != '\n' {
end += 1;
}
chars[start..end]
.iter()
.collect::<String>()
.trim_end()
.to_string()
}
fn the_refusal_of_the_warn(
dialect: &dyn crate::dialect::Dialect,
chars: &[char],
hash_at: usize,
argument: &str,
line: u32,
column: u32,
) -> Result<(), SyntaxError> {
let (kind, mode) = crate::warnings::the_parts_of_the_argument(argument);
if !kind.is_empty() && dialect.the_type_of_the_warning(kind).is_none() {
let source = the_line_of_the_hash(chars, hash_at);
let text = source
.split(',')
.next()
.unwrap_or(source.as_str())
.trim_end();
return Err(SyntaxError::new(
format!("Parameter #1 invalid.\n Specifically: {text}"),
line,
column,
));
}
if !mode.is_empty() && dialect.the_mode_of_the_warning(mode).is_none() {
return Err(SyntaxError::new(
format!("Parameter #2 invalid.\n Specifically: {mode}"),
line,
column,
));
}
Ok(())
}
fn the_source_of_the_chars(chars: &[char], start: usize, end: usize) -> String {
let start = start.min(end).min(chars.len());
let end = end.min(chars.len());
let mut text: String = chars[start..end].iter().collect();
while text.ends_with([' ', '\t', '\r']) {
text.pop();
}
text
}
struct TheOptionsOfASection {
join: Option<String>,
ltrim: bool,
ltrim0: bool,
rtrim0: bool,
comments: bool,
accent: bool,
}
struct TheSection {
options: TheOptionsOfASection,
lines: Vec<String>,
rest: String,
end: usize,
rest_at: usize,
after: usize,
closed: bool,
}
fn the_section_at(
chars: &[char],
start: usize,
line: u32,
dialect: &'static dyn crate::dialect::Dialect,
) -> Result<Option<TheSection>, SyntaxError> {
let mut index = start;
while matches!(chars.get(index), Some(' ') | Some('\t')) {
index += 1;
}
if chars.get(index) != Some(&'(') {
return Ok(None);
}
let column = (index - start + 1) as u32;
index += 1;
let mut end_of_the_line = index;
while end_of_the_line < chars.len() && chars[end_of_the_line] != '\n' {
end_of_the_line += 1;
}
let mut text_of_the_options: String = chars[index..end_of_the_line].iter().collect();
if let Some(at) = text_of_the_options.find(';') {
text_of_the_options.truncate(at);
}
let mut options = TheOptionsOfASection {
join: None,
ltrim: false,
ltrim0: false,
rtrim0: false,
comments: false,
accent: false,
};
let mut the_join = String::new();
let mut the_join_is_there = false;
let mut an_option_is_not_known: Option<String> = None;
let mut a_paren_is_in_the_line = false;
for option in text_of_the_options.split(' ') {
if option.is_empty() {
continue;
}
let (name, argument) = match option.get(..4) {
Some(head) if head.eq_ignore_ascii_case("join") => ("join", Some(&option[4..])),
_ => (option, None),
};
let text = argument.unwrap_or(option);
if text.contains('(') || text.contains(')') {
a_paren_is_in_the_line = true;
}
match name.to_lowercase().as_str() {
"join" => {
the_join_is_there = true;
the_join = argument.unwrap_or("").to_string();
}
"ltrim" => options.ltrim = true,
"ltrim0" => options.ltrim0 = true,
"rtrim0" => options.rtrim0 = true,
"comments" | "comment" | "com" | "c" => options.comments = true,
"`" => options.accent = true,
_ => {
if an_option_is_not_known.is_none() {
an_option_is_not_known = Some(option.to_string());
}
}
}
}
if a_paren_is_in_the_line {
return Ok(None);
}
if let Some(option) = an_option_is_not_known {
return Err(SyntaxError::new(
format!("Invalid option.\n Specifically: {option}"),
line,
column,
));
}
if the_join_is_there {
let the_join: String = the_join.chars().take(15).collect();
options.join = Some(the_escapes_of_a_text(&the_join, options.accent, dialect));
}
let mut lines: Vec<String> = Vec::new();
let mut rest = String::new();
let mut rest_at = end_of_the_line;
let mut end = chars.len();
let mut closed = false;
let mut cursor = end_of_the_line;
while cursor < chars.len() {
let mut start_of_the_line = cursor + 1;
if chars.get(start_of_the_line) == Some(&'\r') {
start_of_the_line += 1;
}
let mut end_of_the_line = start_of_the_line;
while end_of_the_line < chars.len() && chars[end_of_the_line] != '\n' {
end_of_the_line += 1;
}
let text: String = chars[start_of_the_line..end_of_the_line].iter().collect();
let trimmed = text.trim_start_matches([' ', '\t']);
if let Some(after_the_parenthesis) = trimmed.strip_prefix(')') {
rest = after_the_parenthesis.to_string();
rest_at = start_of_the_line + (trimmed.len() - after_the_parenthesis.len());
end = end_of_the_line;
closed = true;
break;
}
lines.push(text);
cursor = end_of_the_line;
}
Ok(Some(TheSection {
options,
lines,
rest,
end,
rest_at,
after: end_of_the_line + 1,
closed,
}))
}
fn the_content_of_the_section(section: &TheSection) -> String {
let mut lines: Vec<String> = Vec::new();
for line in §ion.lines {
let mut text = line.clone();
let mut a_comment_is_the_whole_line = false;
if section.options.comments {
if let Some(at) = text.find(';') {
text.truncate(at);
while text.ends_with([' ', '\t']) {
text.pop();
}
a_comment_is_the_whole_line = text.trim().is_empty();
}
}
if a_comment_is_the_whole_line {
continue;
}
if !section.options.rtrim0 {
while text.ends_with([' ', '\t', '\r']) {
text.pop();
}
}
lines.push(text);
}
let mut indentation = (0usize, ' ');
if !section.options.ltrim && !section.options.ltrim0 {
let first = lines.first().map(|line| line.as_str()).unwrap_or("");
let spaces = first.chars().take_while(|c| *c == ' ').count();
let tabs = first.chars().take_while(|c| *c == '\t').count();
indentation = if spaces > 0 {
(spaces, ' ')
} else {
(tabs, '\t')
};
}
if indentation.0 > 0 {
let (count, character) = indentation;
for line in lines.iter_mut() {
if line.chars().take(count).all(|c| c == character) {
*line = line.chars().skip(count).collect();
}
}
}
if section.options.ltrim {
for line in lines.iter_mut() {
*line = line.trim_start_matches([' ', '\t']).to_string();
}
}
let join = match §ion.options.join {
Some(join) => join.clone(),
None => "\n".to_string(),
};
let mut content = String::new();
for (number, line) in lines.iter().enumerate() {
if number > 0 {
content.push_str(&join);
}
content.push_str(line);
}
content
}
fn the_escapes_of_a_text(
text: &str,
accent: bool,
dialect: &'static dyn crate::dialect::Dialect,
) -> String {
if accent {
return text.to_string();
}
let mut result = String::new();
let mut characters = text.chars();
while let Some(character) = characters.next() {
if character == '`' {
match characters.next() {
Some(next) => result.push(dialect.the_escape_of_the_dialect(next)),
None => result.push('`'),
}
continue;
}
result.push(character);
}
result
}
fn the_tokens_of_a_merged_line(
text: &str,
dialect: &'static dyn crate::dialect::Dialect,
line: u32,
) -> Result<Vec<Token>, SyntaxError> {
let mut tokens = tokenize_the_source(text, dialect, true).map_err(|mut error| {
error.line = line;
error
})?;
if tokens
.last()
.map(|token| token.kind == TokenKind::Eof)
.unwrap_or(false)
{
tokens.pop();
}
for token in tokens.iter_mut() {
token.line = line;
}
Ok(tokens)
}
fn the_section_after_the_line(
chars: &[char],
from: usize,
line: u32,
dialect: &'static dyn crate::dialect::Dialect,
) -> Result<Option<(TheSection, u32)>, SyntaxError> {
let mut at = from;
let mut skipped = 0u32;
loop {
let mut end_of_the_line = at;
while end_of_the_line < chars.len() && chars[end_of_the_line] != '\n' {
end_of_the_line += 1;
}
let text: String = chars[at..end_of_the_line].iter().collect();
let trimmed = text.trim_matches([' ', '\t', '\r']);
if trimmed.is_empty() || trimmed.starts_with(';') {
if end_of_the_line >= chars.len() {
return Ok(None);
}
skipped += 1;
at = end_of_the_line + 1;
continue;
}
let section = the_section_at(chars, at, line + skipped, dialect)?;
return Ok(section.map(|section| (section, skipped)));
}
}
fn the_text_without_the_comment(text: &str) -> &str {
let mut previous = None;
for (at, character) in text.char_indices() {
if character == ';' && (at == 0 || matches!(previous, Some(' ') | Some('\t'))) {
let mut end = at;
while end > 0 && matches!(text.as_bytes().get(end - 1), Some(b' ') | Some(b'\t')) {
end -= 1;
}
return &text[..end];
}
previous = Some(character);
}
text.trim_end_matches([' ', '\t', '\r'])
}
fn the_last_character_is_a_name(text: &str) -> bool {
match text.trim_end().chars().last() {
Some(last) => the_character_is_of_a_name(last, false),
None => false,
}
}
fn the_first_character_is_a_name(text: &str) -> bool {
match text.chars().next() {
Some(first) => the_character_is_of_a_name(first, false),
None => false,
}
}
fn the_line_above_ends_inside_a_text(text: &str) -> bool {
let mut quote: Option<char> = None;
let mut characters = text.chars();
while let Some(character) = characters.next() {
if character == '`' {
characters.next();
continue;
}
match quote {
Some(open) => {
if character == open {
quote = None;
}
}
None => {
if character == '"' || character == '\'' {
quote = Some(character);
}
}
}
}
quote.is_some()
}
pub fn tokenize(source: &str) -> Result<Vec<Token>, SyntaxError> {
tokenize_with(source, crate::dialect::v2())
}
pub fn tokenize_with(
source: &str,
dialect: &'static dyn crate::dialect::Dialect,
) -> Result<Vec<Token>, SyntaxError> {
tokenize_the_source(source, dialect, false)
}
fn tokenize_the_source(
source: &str,
dialect: &'static dyn crate::dialect::Dialect,
the_newlines_join_the_line: bool,
) -> Result<Vec<Token>, SyntaxError> {
let source = source.strip_prefix('\u{feff}').unwrap_or(source);
let chars: Vec<char> = source.chars().collect();
let marks = dialect.accepts_a_hash_a_dollar_and_an_at_in_a_name();
let operators = the_operators_of(dialect);
let mut tokens: Vec<Token> = Vec::new();
let mut index = 0usize;
let mut line = 1u32;
let mut column = 1u32;
let mut spaced = false;
let mut the_starts_of_the_lines: Vec<usize> = vec![0];
let mut the_line_with_code: Option<(u32, String)> = None;
let mut the_token_start_of_the_line_with_code = 0usize;
let mut the_start_of_the_current_line = 0usize;
let mut the_start_of_the_current_line_tokens = 0usize;
let mut a_section_was_merged = false;
let mut the_hotstrings_read_code = true;
while index < chars.len() {
let current = chars[index];
let before = spaced;
spaced = false;
if current == '\n' {
if the_newlines_join_the_line {
index += 1;
line += 1;
column = 1;
spaced = true;
continue;
}
if tokens.len() > the_start_of_the_current_line_tokens && !a_section_was_merged {
let mut text: String = chars[the_start_of_the_current_line..index].iter().collect();
while text.ends_with(['\r', ' ', '\t']) {
text.pop();
}
the_line_with_code = Some((line, text));
the_token_start_of_the_line_with_code = the_start_of_the_current_line_tokens;
}
if let Some((section, the_skipped_lines)) =
the_section_after_the_line(&chars, index + 1, line + 1, dialect)?
{
let the_line_above = the_line_with_code.clone();
if let Some((the_number, text_of_the_line)) = &the_line_above {
let the_number = *the_number;
let the_directive_is_the_line_above = tokens
.iter()
.skip(the_token_start_of_the_line_with_code)
.any(|token| token.kind == TokenKind::Directive);
if the_directive_is_the_line_above {
let content = the_content_of_the_section(§ion);
let mut argument = String::new();
if tokens
.last()
.map(|token| {
token.kind == TokenKind::String && token.line == the_number
})
.unwrap_or(false)
{
argument = tokens.pop().map(|token| token.value).unwrap_or_default();
}
argument.push_str(&content);
argument.push_str(§ion.rest);
tokens.push(Token {
kind: TokenKind::String,
value: argument,
line: the_number,
column: 1,
spaced: false,
quote: None,
});
index = section.end;
line += the_skipped_lines + section.lines.len() as u32 + 2;
column = 1;
a_section_was_merged = true;
continue;
}
let text_of_the_line = the_text_without_the_comment(text_of_the_line).to_string();
let content = the_content_of_the_section(§ion);
let mut merged = text_of_the_line;
let a_space = !content.is_empty()
&& !the_line_above_ends_inside_a_text(&merged)
&& the_last_character_is_a_name(&merged)
&& the_first_character_is_a_name(&content);
if a_space {
merged.push(' ');
}
if !section.closed {
let mut text = merged;
let rest: String = chars[section.after..].iter().collect();
text.push_str(rest.trim_end_matches(['\r', '\n']));
return Err(SyntaxError::new(
format!("Missing \")\"\n Specifically: {text}"),
the_number,
1,
));
}
merged.push_str(&content);
merged.push_str(§ion.rest);
tokens.truncate(the_token_start_of_the_line_with_code);
let the_merged_tokens =
the_tokens_of_a_merged_line(&merged, dialect, the_number)?;
tokens.extend(the_merged_tokens);
the_line_with_code = Some((the_number, merged));
the_start_of_the_current_line_tokens = the_token_start_of_the_line_with_code;
a_section_was_merged = true;
index = section.end;
line += the_skipped_lines + section.lines.len() as u32 + 2;
column = 1;
continue;
}
}
tokens.push(Token {
kind: TokenKind::Newline,
value: "\n".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
line += 1;
column = 1;
the_starts_of_the_lines.push(index);
the_start_of_the_current_line = index;
the_start_of_the_current_line_tokens = tokens.len();
a_section_was_merged = false;
continue;
}
if current == ' ' || current == '\t' || current == '\r' {
index += 1;
column += 1;
spaced = true;
continue;
}
if current == '#' {
if tokens.len() == the_start_of_the_current_line_tokens
&& the_line_from_has_the_two_colons(&chars, index)
{
tokens.push(Token {
kind: TokenKind::Op,
value: "#".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
column += 1;
continue;
}
if tokens.iter().any(|token| token.line == line) {
tokens.push(Token {
kind: TokenKind::Op,
value: "#".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
column += 1;
continue;
}
let (start_line, start_column) = (line, column);
let hash_at = index;
index += 1;
column += 1;
let mut name = String::new();
while index < chars.len() && the_character_is_of_a_name(chars[index], marks) {
name.push(chars[index]);
index += 1;
column += 1;
}
if name.is_empty() || name.starts_with(|character: char| character.is_ascii_digit()) {
return Err(SyntaxError::new(
format!(
"This line does not contain a recognized action.\n Specifically: {}",
the_line_of_the_hash(&chars, hash_at)
),
start_line,
start_column,
));
}
let mut argument = String::new();
while index < chars.len() && chars[index] != '\n' {
argument.push(chars[index]);
index += 1;
column += 1;
}
if name.eq_ignore_ascii_case("hotstring") {
let argument = argument.trim();
if argument.is_empty() {
the_hotstrings_read_code = true;
} else {
for option in argument.split_whitespace() {
if option.eq_ignore_ascii_case("x") {
the_hotstrings_read_code = true;
} else if option.eq_ignore_ascii_case("x0") {
the_hotstrings_read_code = false;
}
}
}
}
if !dialect.carries_directive(&name) {
return Err(SyntaxError::new(
format!(
"This line does not contain a recognized action.\n Specifically: #{name}{}",
argument.trim_end()
),
start_line,
start_column,
));
}
if name.eq_ignore_ascii_case("warn") {
the_refusal_of_the_warn(
dialect,
&chars,
hash_at,
&argument,
start_line,
start_column,
)?;
}
tokens.push(Token {
kind: TokenKind::Directive,
value: name.to_lowercase(),
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
let argument = argument.trim().to_string();
if !argument.is_empty() {
tokens.push(Token {
kind: TokenKind::String,
value: argument,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
}
continue;
}
if current == '/' && chars.get(index + 1) == Some(&'*') {
index += 2;
column += 2;
while index < chars.len() {
if chars[index] == '*' && chars.get(index + 1) == Some(&'/') {
index += 2;
column += 2;
break;
}
if chars[index] == '\n' {
index += 1;
line += 1;
column = 1;
the_starts_of_the_lines.push(index);
continue;
}
index += 1;
column += 1;
}
spaced = true;
continue;
}
if current == ';' {
if before || !tokens.iter().any(|token| token.line == line) {
while index < chars.len() && chars[index] != '\n' {
index += 1;
}
continue;
}
tokens.push(Token {
kind: TokenKind::Op,
value: ";".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
column += 1;
continue;
}
if current == '\'' && dialect.the_single_quote_is_not_a_text() {
let (start_line, start_column) = (line, column);
let mut end = index;
while end < chars.len() && !the_character_ends_the_symbol_of_the_v1(chars[end]) {
end += 1;
}
let symbol: String = chars[index..end].iter().collect();
column += (end - index) as u32;
index = end;
tokens.push(Token {
kind: TokenKind::Name,
value: symbol,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
if current == '"' || current == '\'' {
let (start_line, start_column) = (line, column);
let quote = current;
let quote_index = index;
index += 1;
column += 1;
let mut buffer = String::new();
let mut the_text_of_the_section = String::new();
loop {
while index < chars.len() && chars[index] != '\n' {
if chars[index] == '"'
&& dialect.the_quote_escapes_the_quote()
&& index + 1 < chars.len()
&& chars[index + 1] == '"'
{
buffer.push('"');
index += 2;
column += 2;
continue;
}
if chars[index] == quote {
break;
}
if chars[index] == '`' && index + 1 < chars.len() {
let next = chars[index + 1];
if next == quote && !dialect.the_escape_protects_the_quote_of_a_text() {
index += 1;
column += 1;
break;
}
buffer.push(dialect.the_escape_of_the_dialect(next));
index += 2;
column += 2;
continue;
}
buffer.push(chars[index]);
index += 1;
column += 1;
}
if index >= chars.len() || chars[index] != '\n' {
break;
}
if the_newlines_join_the_line {
buffer.push('\n');
index += 1;
line += 1;
column = 1;
continue;
}
let Some((section, the_skipped_lines)) =
the_section_after_the_line(&chars, index + 1, line + 1, dialect)?
else {
break;
};
if !section.closed {
let start_of_the_line = quote_index - (start_column as usize - 1);
let mut text: String = chars[start_of_the_line..index].iter().collect();
let rest: String = chars[section.after..].iter().collect();
text.push_str(rest.trim_end_matches(['\r', '\n']));
return Err(SyntaxError::new(
format!("Missing \")\"\n Specifically: {text}"),
start_line,
start_column,
));
}
if the_text_of_the_section.is_empty() {
let without_the_comment = the_text_without_the_comment(&buffer).to_string();
buffer = without_the_comment;
}
let content = the_content_of_the_section(§ion);
the_text_of_the_section = content.clone();
buffer.push_str(&the_escapes_of_a_text(
&content,
section.options.accent,
dialect,
));
let section_start = index + 1;
index = section.rest_at;
let mut cursor = section_start;
while cursor < section.end {
the_starts_of_the_lines.push(cursor);
match chars[cursor..section.end].iter().position(|c| *c == '\n') {
Some(at) => cursor += at + 1,
None => break,
}
}
line += the_skipped_lines + section.lines.len() as u32 + 2;
column = 1;
}
if index >= chars.len() || chars[index] == '\n' {
let start = quote_index - (start_column as usize - 1);
let mut end = index;
while end < chars.len() && chars[end] != '\n' {
end += 1;
}
let mut text = the_source_of_the_chars(&chars, start, end);
if !the_text_of_the_section.is_empty() {
let mut end_of_the_line = start;
while end_of_the_line < chars.len() && chars[end_of_the_line] != '\n' {
end_of_the_line += 1;
}
text = the_source_of_the_chars(&chars, start, end_of_the_line);
text.push_str(&the_text_of_the_section);
}
if dialect.the_text_that_is_not_closed_is_named_alone() {
let mut end = index;
while end < chars.len() && chars[end] != '\n' {
end += 1;
}
let text = the_source_of_the_chars(&chars, quote_index, end);
if dialect.the_quote_that_is_not_closed_is_an_error() {
return Err(SyntaxError::new(
format!("Missing close-quote\n Specifically: {text}"),
start_line,
start_column,
));
}
tokens.push(Token {
kind: TokenKind::UnclosedText,
value: text,
line: start_line,
column: start_column,
spaced: before,
quote: Some(quote),
});
continue;
}
return Err(SyntaxError::new(
format!("Missing \"{quote}\"\n Specifically: {text}"),
start_line,
start_column,
));
}
index += 1;
column += 1;
tokens.push(Token {
kind: TokenKind::String,
value: buffer,
line: start_line,
column: start_column,
spaced: before,
quote: Some(quote),
});
continue;
}
if current.is_ascii_digit()
|| (current == '.' && the_point_begins_a_number(&chars, index, &tokens))
{
let (start_line, start_column) = (line, column);
let start = index;
index =
the_end_of_the_number(&chars, index, dialect.the_exponent_needs_a_point(), marks);
column += (index - start) as u32;
if index < chars.len()
&& (the_character_is_a_letter(chars[index], marks)
|| (chars[index] == '`' && dialect.the_escape_may_be_in_a_name()))
{
let number_end = index;
let mut name = chars[start..index].iter().collect::<String>();
if chars[index] == '`' {
let escape = the_name_with_the_escape(&chars, index, marks, dialect);
if let Some(operator) = escape.operator {
column += (escape.end - number_end) as u32;
index = escape.end;
tokens.push(Token {
kind: TokenKind::Number,
value: chars[start..number_end].iter().collect(),
quote: None,
line: start_line,
column: start_column,
spaced: before,
});
tokens.push(Token {
kind: TokenKind::Op,
value: operator,
quote: None,
line: start_line,
column: start_column + (number_end - start) as u32,
spaced: false,
});
continue;
}
name.push_str(&escape.name);
index = escape.end;
}
while index < chars.len() && the_character_is_of_a_name(chars[index], marks) {
name.push(chars[index]);
index += 1;
}
if chars.get(index) == Some(&':') {
column += (index - number_end) as u32;
tokens.push(Token {
kind: TokenKind::Name,
value: name,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
if dialect.the_number_may_be_followed_by_a_name() {
column += (index - number_end) as u32;
tokens.push(Token {
kind: TokenKind::Name,
value: name,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
return Err(SyntaxError::new(
format!(
"This variable name starts with a number, which is not allowed:\n{}",
the_name_of_a_message(&name)
),
start_line,
start_column,
));
}
if dialect.the_name_may_not_carry_an_illegal_character()
&& chars.get(index) == Some(&'"')
{
let text: String = chars[start..index].iter().collect();
if text.contains('.') {
return Err(SyntaxError::new(
format!(
"This character is not allowed here.\n Specifically: {}",
the_source_of_a_text(&chars, index)
),
start_line,
start_column,
));
}
let end = the_end_of_a_symbol_that_carries_a_quote(&chars, index);
let word: String = chars[start..end].iter().collect();
column += (end - index) as u32;
index = end;
tokens.push(Token {
kind: TokenKind::Name,
value: word,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
tokens.push(Token {
kind: TokenKind::Number,
value: chars[start..index].iter().collect(),
quote: None,
line: start_line,
column: start_column,
spaced: before,
});
continue;
}
if the_character_is_a_letter(current, marks)
|| (current == '`' && dialect.the_escape_may_be_in_a_name())
{
let (start_line, start_column) = (line, column);
let start = index;
let name = the_name_with_the_escape(&chars, index, marks, dialect);
index = name.end;
column += (index - start) as u32;
if !name.has_an_escape
&& dialect.the_name_may_not_carry_an_illegal_character()
&& chars.get(index) == Some(&'"')
{
let end = the_end_of_a_symbol_that_carries_a_quote(&chars, index);
let word: String = chars[start..end].iter().collect();
column += (end - index) as u32;
index = end;
tokens.push(Token {
kind: TokenKind::Name,
value: word,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
let word = name.name;
let lower = word.to_lowercase();
let kind = if KEYWORDS.contains(&lower.as_str())
&& !(dialect.the_words_may_be_the_name_of_a_variable()
&& matches!(lower.as_str(), "is" | "as" | "in" | "contains" | "super"))
{
TokenKind::Keyword
} else {
TokenKind::Name
};
tokens.push(Token {
kind,
value: word,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
if let Some(operator) = name.operator {
tokens.push(Token {
kind: TokenKind::Op,
value: operator,
quote: None,
line: start_line,
column: start_column,
spaced: false,
});
}
continue;
}
if current == '\\' && dialect.accepts_a_bare_backslash() {
tokens.push(Token {
kind: TokenKind::Name,
value: "\\".into(),
quote: None,
line,
column,
spaced: before,
});
index += 1;
column += 1;
continue;
}
if dialect.the_hotstrings_are_of_the_v2()
&& current == ':'
&& the_line_from_has_the_two_colons(&chars, index)
&& tokens.len() == the_start_of_the_current_line_tokens
{
let start_column = column;
let start = index;
while index < chars.len() && chars[index] != '\n' {
index += 1;
column += 1;
}
let text: String = chars[start..index].iter().collect();
let mut after = index;
if after < chars.len() && chars[after] == '\r' {
after += 1;
}
let mut the_text = text;
if after < chars.len() && chars[after] == '\n' {
let mut at = after + 1;
while at < chars.len() && (chars[at] == ' ' || chars[at] == '\t') {
at += 1;
}
if at < chars.len() && chars[at] == '(' && the_rest_of_the_line_is_blank(&chars, at + 1)
{
let content_start = at + 1;
let mut line_begin = content_start;
let mut closing: Option<usize> = None;
while line_begin <= chars.len() {
let mut line_end = line_begin;
while line_end < chars.len() && chars[line_end] != '\n' {
line_end += 1;
}
let line_text: String = chars[line_begin..line_end].iter().collect();
if line_begin != content_start && line_text.trim() == ")" {
closing = Some(line_begin);
break;
}
if line_end >= chars.len() {
break;
}
line_begin = line_end + 1;
}
if let Some(close_start) = closing {
let content: String = chars[content_start..close_start].iter().collect();
the_text.push('\n');
the_text.push_str(content.trim_end_matches(['\r', '\n']));
let consumed = chars[index..close_start]
.iter()
.filter(|character| **character == '\n')
.count() as u32;
line += consumed;
index = close_start;
while index < chars.len() && chars[index] != '\n' {
index += 1;
}
column = 1;
}
}
}
if the_hotstrings_read_code
&& the_hotstring_of_the_v2_has_a_block(&the_text)
&& the_block_of_the_hotstring_follows(&chars, &the_text, index)
{
let mut end = index;
let mut depth = 0i32;
let mut at = end;
while at < chars.len() {
let mut line_end = at;
while line_end < chars.len() && chars[line_end] != '\n' {
line_end += 1;
}
for character in &chars[at..line_end] {
match character {
'{' => depth += 1,
'}' => depth -= 1,
_ => {}
}
}
if depth <= 0 && chars[at..line_end].contains(&'}') {
end = line_end;
break;
}
if line_end >= chars.len() {
end = line_end;
break;
}
at = line_end + 1;
}
let block: String = chars[index..end].iter().collect();
the_text.push_str(block.trim_end_matches(['\r', '\n']));
let consumed = chars[index..end]
.iter()
.filter(|character| **character == '\n')
.count() as u32;
line += consumed;
index = end;
while index < chars.len() && chars[index] != '\n' {
index += 1;
}
column = 1;
}
tokens.push(Token {
kind: TokenKind::String,
value: the_text,
line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
let mut matched = false;
for operator in operators.iter().copied() {
if matches!(operator, "++" | "--")
&& !dialect.the_increment_after_a_number_is_an_operator()
&& tokens
.last()
.map(|token| token.kind == TokenKind::Number)
.unwrap_or(false)
{
continue;
}
if chars[index..].starts_with(&operator.chars().collect::<Vec<char>>()[..]) {
tokens.push(Token {
kind: TokenKind::Op,
value: (*operator).to_string(),
quote: None,
line,
column,
spaced: before,
});
index += operator.chars().count();
column += operator.chars().count() as u32;
matched = true;
break;
}
}
if !matched {
if current == '$' && tokens.len() == the_start_of_the_current_line_tokens {
tokens.push(Token {
kind: TokenKind::Op,
value: "$".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
column += 1;
continue;
}
return Err(SyntaxError::new(
format!("unexpected character {current:?}"),
line,
column,
));
}
}
tokens.push(Token {
kind: TokenKind::Eof,
value: String::new(),
line,
column,
spaced: false,
quote: None,
});
Ok(tokens)
}
fn the_point_begins_a_number(chars: &[char], index: usize, tokens: &[Token]) -> bool {
if !chars
.get(index + 1)
.is_some_and(|next| next.is_ascii_digit())
{
return false;
}
match tokens.last() {
Some(token) => !the_token_ends_a_value(token),
None => true,
}
}
fn the_token_ends_a_value(token: &Token) -> bool {
match token.kind {
TokenKind::Name | TokenKind::Number | TokenKind::String => true,
TokenKind::Op => matches!(token.value.as_str(), ")" | "]" | "}"),
_ => false,
}
}
fn the_digits_of_an_exponent_follow(chars: &[char], from: usize) -> bool {
let mut at = from;
if matches!(chars.get(at), Some(&'+') | Some(&'-')) {
at += 1;
}
chars.get(at).is_some_and(|digit| digit.is_ascii_digit())
}
fn the_end_of_the_number(
chars: &[char],
start: usize,
the_exponent_needs_a_point: bool,
marks: bool,
) -> usize {
if chars.get(start) == Some(&'0')
&& matches!(chars.get(start + 1), Some(&'x') | Some(&'X'))
&& chars
.get(start + 2)
.is_some_and(|digit| digit.is_ascii_hexdigit())
{
let mut index = start + 2;
while index < chars.len() && chars[index].is_ascii_hexdigit() {
index += 1;
}
return index;
}
let mut index = start;
while index < chars.len() && chars[index].is_ascii_digit() {
index += 1;
}
let mut the_number_has_a_point = false;
if chars.get(index) == Some(&'.') {
let the_point_is_in_the_number = match chars.get(index + 1) {
Some('=') => false,
Some(letter) if (*letter == 'e' || *letter == 'E')
&& the_digits_of_an_exponent_follow(chars, index + 2) =>
{
true
}
Some(character) if the_character_is_a_letter(*character, marks) => false,
_ => true,
};
if the_point_is_in_the_number {
the_number_has_a_point = true;
index += 1;
while index < chars.len() && chars[index].is_ascii_digit() {
index += 1;
}
}
}
if (!the_exponent_needs_a_point || the_number_has_a_point)
&& matches!(chars.get(index), Some(&'e') | Some(&'E'))
{
let mut after = index + 1;
if matches!(chars.get(after), Some(&'+') | Some(&'-')) {
after += 1;
}
if chars.get(after).is_some_and(|digit| digit.is_ascii_digit()) {
while after < chars.len() && chars[after].is_ascii_digit() {
after += 1;
}
return after;
}
}
index
}
#[cfg(test)]
mod tests {
#[test]
fn the_null_assignment_is_a_token_of_the_v2_1() {
let tokens = tokenize_with("z ??= 5", crate::dialect::v2_1()).expect("tokens");
assert!(
tokens
.iter()
.any(|token| token.kind == TokenKind::Op && token.value == "??="),
"{tokens:?}"
);
let tokens = tokenize_with("z ??= 5", crate::dialect::v2()).expect("tokens");
assert!(
!tokens
.iter()
.any(|token| token.kind == TokenKind::Op && token.value == "??="),
"{tokens:?}"
);
}
use super::*;
fn kinds(source: &str) -> Vec<TokenKind> {
tokenize(source)
.expect("parses")
.iter()
.map(|t| t.kind)
.collect()
}
#[test]
fn a_string_is_one_token() {
let tokens = tokenize("Send \"hello world\"").expect("parses");
assert_eq!(tokens[0].value, "Send");
assert_eq!(tokens[1].kind, TokenKind::String);
assert_eq!(tokens[1].value, "hello world");
}
#[test]
fn a_comment_is_not_a_token() {
assert_eq!(
kinds("Send \"a\" ; this is a comment"),
vec![TokenKind::Name, TokenKind::String, TokenKind::Eof]
);
}
#[test]
fn an_unterminated_string_is_an_error() {
let error = tokenize("Send \"never closed").expect_err("fails");
assert_eq!(error.line, 1);
assert_eq!(
error.message,
"Missing \"\"\"\n Specifically: Send \"never closed"
);
let error = tokenize("Send 'never closed").expect_err("fails");
assert_eq!(error.line, 1);
assert_eq!(
error.message,
"Missing \"'\"\n Specifically: Send 'never closed"
);
}
#[test]
fn a_text_is_between_double_quotes_or_between_single_quotes() {
let tokens = tokenize("x := 'hello'").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::String);
assert_eq!(tokens[2].value, "hello");
assert_eq!(tokens[2].quote, Some('\''));
let tokens = tokenize("x := \"hello\"").expect("parses");
assert_eq!(tokens[2].quote, Some('"'));
}
#[test]
fn the_escape_of_the_dialect_is_the_backtick_in_the_two_kinds_of_text() {
let tokens = tokenize("x := 'a`nb'").expect("parses");
assert_eq!(tokens[2].value, "a\nb");
let tokens = tokenize("x := \"a`nb\"").expect("parses");
assert_eq!(tokens[2].value, "a\nb");
}
#[test]
fn a_text_does_not_cross_a_line() {
let error = tokenize("x := \"a\nb\"").expect_err("fails");
assert_eq!(error.line, 1);
assert_eq!(error.message, "Missing \"\"\"\n Specifically: x := \"a");
}
#[test]
fn a_number_keeps_its_point() {
let tokens = tokenize("x := 3.5").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "3.5");
}
#[test]
fn the_hex_of_v2_is_one_token() {
let tokens = tokenize("x := 0xE9").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "0xE9");
}
#[test]
fn the_exponent_of_v2_is_one_token() {
let tokens = tokenize("x := 1e3").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "1e3");
}
#[test]
fn a_number_has_one_point() {
let tokens = tokenize("x := 1.5.2").expect("parses");
assert_eq!(tokens[2].value, "1.5");
assert_eq!(tokens[3].value, ".");
}
#[test]
fn a_name_that_begins_with_a_number_is_an_error() {
let error = tokenize("x := 12abc").expect_err("fails");
assert_eq!(
error.message,
"This variable name starts with a number, which is not allowed:\n\"12abc\""
);
assert_eq!(error.line, 1);
}
#[test]
fn the_longest_operator_wins() {
let tokens = tokenize("x := a >>>= b").expect("parses");
assert_eq!(tokens[3].value, ">>>=");
}
#[test]
fn the_assignment_operators_of_v2_are_one_token() {
for operator in ["|=", "^=", "&=", "~="] {
let tokens = tokenize(&format!("x {operator} 2")).expect("parses");
assert_eq!(tokens[1].value, operator);
}
}
#[test]
fn a_decrement_after_a_number_is_a_subtraction() {
let tokens = tokenize("x := 5--3").expect("parses");
assert_eq!(tokens[3].value, "-");
}
#[test]
fn a_decrement_after_a_name_is_a_decrement() {
let tokens = tokenize("x--").expect("parses");
assert_eq!(tokens[1].value, "--");
}
#[test]
fn a_directive_is_a_directive() {
let tokens = tokenize("#Requires AutoHotkey v2.0.19\n").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Directive);
assert_eq!(tokens[0].value, "requires");
assert_eq!(tokens[1].kind, TokenKind::String);
assert_eq!(tokens[1].value, "AutoHotkey v2.0.19");
}
#[test]
fn the_word_for_is_a_keyword() {
let tokens = tokenize("for x in y").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Keyword);
assert_eq!(tokens[0].value, "for");
assert_eq!(tokens[2].kind, TokenKind::Keyword);
assert_eq!(tokens[2].value, "in");
}
#[test]
fn the_word_operators_are_keywords() {
let tokens = tokenize("a is b and c").expect("parses");
assert_eq!(tokens[1].kind, TokenKind::Keyword);
assert_eq!(tokens[1].value, "is");
assert_eq!(tokens[3].kind, TokenKind::Keyword);
assert_eq!(tokens[3].value, "and");
}
#[test]
fn the_script_may_begin_with_the_byte_order_mark() {
let tokens = tokenize("\u{feff}x := 1\n").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Name);
assert_eq!(tokens[0].value, "x");
}
#[test]
fn the_byte_order_mark_is_a_letter_of_a_name() {
let tokens = tokenize("x\u{feff}y := 1\n").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Name);
assert_eq!(tokens[0].value, "x\u{feff}y");
}
#[test]
fn the_name_that_begins_with_a_number_and_a_mark_is_refused() {
let error = tokenize("1\u{feff}x := 1\n").expect_err("does not lex");
assert_eq!(
error.message,
"This variable name starts with a number, which is not allowed:\n\"1?x\""
);
}
#[test]
fn the_directive_without_a_name_is_a_line_that_holds_no_action() {
let error = tokenize("#\n").expect_err("does not lex");
assert_eq!(
error.message,
"This line does not contain a recognized action.\n Specifically: #"
);
}
#[test]
fn the_directive_with_a_number_is_a_line_that_holds_no_action() {
let error = tokenize("#123\n").expect_err("does not lex");
assert_eq!(
error.message,
"This line does not contain a recognized action.\n Specifically: #123"
);
}
#[test]
fn the_directive_that_is_not_known_is_a_line_that_holds_no_action() {
let error = tokenize("#Foo Bar\n").expect_err("does not lex");
assert_eq!(
error.message,
"This line does not contain a recognized action.\n Specifically: #Foo Bar"
);
}
#[test]
fn the_semicolon_is_a_comment_only_when_it_is_apart() {
let tokens = tokenize("x := 1;c").expect("parses");
assert_eq!(tokens[3].kind, TokenKind::Op);
assert_eq!(tokens[3].value, ";");
let tokens = tokenize("x := 1 ;c").expect("parses");
assert_eq!(tokens.len(), 4);
assert_eq!(tokens[3].kind, TokenKind::Eof);
let tokens = tokenize(";c").expect("parses");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0].kind, TokenKind::Eof);
}
#[test]
fn the_point_of_a_number_is_not_the_point_of_a_member() {
let tokens = tokenize("x := 1.ToString").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "1");
assert_eq!(tokens[3].value, ".");
assert_eq!(tokens[4].value, "ToString");
let tokens = tokenize("x := 1.").expect("parses");
assert_eq!(tokens[2].value, "1.");
}
#[test]
fn the_three_way_comparison_is_not_a_token() {
let tokens = tokenize("x := 1 === 1").expect("parses");
assert_eq!(tokens[3].value, "==");
assert_eq!(tokens[4].value, "=");
}
fn values(source: &str) -> Vec<String> {
tokenize(source)
.expect("parses")
.into_iter()
.filter(|token| token.kind != TokenKind::Newline && token.kind != TokenKind::Eof)
.map(|token| token.value)
.collect()
}
fn text(source: &str) -> String {
tokenize(source)
.expect("parses")
.into_iter()
.find(|token| token.kind == TokenKind::String)
.map(|token| token.value)
.expect("a text")
}
#[test]
fn a_section_outside_a_text_is_the_source_text_of_the_line() {
assert_eq!(values("Var :=\n(\na\nb\n)"), ["Var", ":=", "a", "b"]);
assert_eq!(values("n :=\n(\n1\n+ 2\n)"), ["n", ":=", "1", "+", "2"]);
assert_eq!(values("op :\n(Join\n= 5\n)"), ["op", ":=", "5"]);
}
#[test]
fn a_section_that_follows_a_text_is_a_part_of_that_text() {
let source = "Var := \"\n(\nA line.\nBy default.\n)\"";
assert_eq!(values(source).len(), 3);
assert_eq!(text(source), "A line.\nBy default.");
let source = "Var :=\n(\n\"A line.\nA line of text.\"\n)";
assert_eq!(text(source), "A line.\nA line of text.");
}
#[test]
fn the_comment_of_the_line_above_a_section_is_of_no_account() {
assert_eq!(
text("Var := \"This is a string, ; Comment.\"\n(\n followed by a comment.\n)\""),
"This is a string,followed by a comment."
);
assert_eq!(
text("Var := \"This is a string, ; Comment.\n(\n followed by a comment.\n)\""),
"This is a string,followed by a comment."
);
assert_eq!(text("Var := \"a ; b\"\n(\nc\n)\""), "ac");
assert_eq!(values("x := 1 ; comment\n(Join\n+ 2\n)"), ["x", ":=", "1", "+", "2"]);
}
#[test]
fn the_blank_lines_and_the_comments_before_a_section_are_of_no_account() {
let source = "gap := \" ; Comment.\n; Comment.\n\n( LTrim Join ; Comment.\n ; This is not a comment; it is literal.\n)\"";
assert_eq!(text(source), "; This is not a comment; it is literal.");
assert_eq!(
values("Var :=\n(\n obj\n)\n/* Comment is ignored */\n(\n .prop\n)"),
["Var", ":=", "obj", ".", "prop"]
);
}
#[test]
fn a_block_comment_is_not_a_token() {
assert_eq!(values("x := 1 /* a comment */ + 2"), ["x", ":=", "1", "+", "2"]);
let tokens = tokenize("x := 1 /* a\ncomment */ + 2").expect("parses");
assert_eq!(tokens.last().map(|token| token.line), Some(2));
}
#[test]
fn the_options_of_a_section_are_read() {
assert_eq!(text("Var := \"\n( LTrim\n One.\n Two.\n)\""), "One.\nTwo.");
assert_eq!(
text("Var := \"\n( LTrim0\n One.\n Two.\n)\""),
" One.\n Two."
);
assert_eq!(text("Var := \"\n( RTrim0\none \ntwo \n)\""), "one \ntwo ");
assert_eq!(
text("Var := \"\n(\n One.\n Two.\n Three.\n)\""),
"One.\nTwo.\n Three."
);
assert_eq!(text("Var := \"\n(\none\n\ntwo\n)\""), "one\n\ntwo");
}
#[test]
fn the_option_comments_takes_the_comments_of_the_lines_away() {
assert_eq!(
text("Var := \"\n(c\none ; trailing comment\n; only a comment\ntwo\n)\""),
"one\ntwo"
);
assert_eq!(text("Var := \"\n(C\none ; c\ntwo\n)\""), "one\ntwo");
assert_eq!(
text("Var := \"\n(Comments RTrim0\none ; cmt\ntwo \n)\""),
"one\ntwo "
);
}
#[test]
fn the_string_of_the_option_join_is_the_delimiter() {
assert_eq!(text("Var := \"\n( Join|\none\ntwo\n)\""), "one|two");
assert_eq!(text("Var := \"\n(Join##\none\ntwo\n)\""), "one##two");
assert_eq!(text("Var := \"\n(Join%\none\ntwo\n)\""), "one%two");
assert_eq!(
text("Var := \"\n(Join0123456789abcdefGHI\none\ntwo\n)\""),
"one0123456789abcdetwo"
);
assert_eq!(
values("name :=\n(JoinrL\nSt\nen(\"abcde\")\n)"),
["name", ":=", "StrLen", "(", "abcde", ")"]
);
}
#[test]
fn the_accent_of_a_section_makes_the_backtick_a_literal() {
assert_eq!(text("Var := \"\n(`\nesc`ttab\n)\""), "esc`ttab");
assert_eq!(text("Var := \"\n(\nesc`ttab\n)\""), "esc\ttab");
assert_eq!(
text("Var := \"\n(\n`)Escaped closing paren.\n)\""),
")Escaped closing paren."
);
}
#[test]
fn a_section_may_follow_a_section_on_the_same_logical_line() {
let tokens = tokenize("arr := Array(\n(Join,\na\nb\n)\n(Join,\nc\nd\n)\n)")
.expect("parses");
let values: Vec<String> = tokens
.iter()
.filter(|token| token.kind != TokenKind::Newline && token.kind != TokenKind::Eof)
.map(|token| token.value.clone())
.collect();
assert_eq!(values, ["arr", ":=", "Array", "(", "a", ",", "b", "c", ",", "d", ")"]);
let b = tokens
.iter()
.position(|token| token.value == "b")
.expect("the name");
assert_eq!(tokens[b + 1].value, "c");
assert!(tokens[b + 1].spaced);
}
#[test]
fn the_lines_of_a_text_that_hold_sections_are_counted() {
assert_eq!(
values("Var := \"\n(\na\n)\n(Join\nb\n)\"\nVar2 :=\n(\n1\n+ 2\n)"),
["Var", ":=", "ab", "Var2", ":=", "1", "+", "2"]
);
}
#[test]
fn a_section_that_is_not_closed_names_the_text_that_it_merged() {
let error = tokenize("Var := \"\n(\nA line of text.\n").expect_err("does not parse");
assert_eq!(
error.message,
"Missing \")\"\n Specifically: Var := \"A line of text."
);
let error = tokenize("Var :=\n(\nA line of text.\nAnother line.").expect_err("does not parse");
assert_eq!(
error.message,
"Missing \")\"\n Specifically: Var :=A line of text.\nAnother line."
);
}
#[test]
fn the_option_that_v2_does_not_know_is_refused() {
let error = tokenize("Var := \"\n(Bogus\nA line.\n)\"").expect_err("does not parse");
assert_eq!(error.message, "Invalid option.\n Specifically: Bogus");
}
}