use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec;
use alloc::vec::Vec;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenKind {
Name,
Keyword,
Number,
String,
UnclosedText,
Op,
Directive,
Newline,
Eof,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub value: String,
pub line: u32,
pub column: u32,
pub spaced: bool,
pub quote: Option<char>,
}
pub const KEYWORDS: &[&str] = &[
"if", "else", "switch", "case", "return", "exitapp", "while", "until", "loop", "try", "catch",
"finally", "throw", "for", "break", "continue", "goto", "true", "false", "and", "or", "not",
"is", "in", "contains", "as", "class", "static", "global", "local", "super",
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TheRefusal {
Variable,
Syntax,
Future,
Func,
Class,
}
fn the_end_of_a_symbol_that_carries_a_quote(chars: &[char], start: usize) -> usize {
let mut end = start;
while end < chars.len() && !the_character_ends_a_symbol(chars[end]) {
end += 1;
}
end
}
fn the_character_ends_a_symbol(character: char) -> bool {
character.is_whitespace()
|| matches!(
character,
',' | ')'
| '}'
| '{'
| ':'
| '.'
| ';'
| '='
| '!'
| '<'
| '>'
| '&'
| '|'
| '^'
| '+'
| '-'
| '*'
| '/'
| '%'
| '~'
| '?'
| '('
| '['
)
}
pub fn the_character_ends_the_symbol_of_the_v1(character: char) -> bool {
character.is_whitespace()
|| matches!(
character,
',' | ')'
| '('
| '}'
| '{'
| ']'
| '['
| ':'
| '='
| '!'
| '<'
| '>'
| '&'
| '|'
| '^'
| '+'
| '-'
| '*'
| '/'
| '~'
| '?'
)
}
fn the_source_of_a_text(chars: &[char], start: usize) -> String {
let mut end = start + 1;
while end < chars.len() && chars[end] != '"' && chars[end] != '\n' {
end += 1;
}
if end < chars.len() && chars[end] == '"' {
end += 1;
}
chars[start..end].iter().collect()
}
pub fn the_refusal_of_a_word(word: &str) -> Option<TheRefusal> {
let word = word.to_lowercase();
Some(match word.as_str() {
"if" | "else" | "switch" | "case" | "return" | "while" | "until" | "loop" | "try"
| "catch" | "finally" | "throw" | "for" | "break" | "continue" | "goto" | "static"
| "global" | "local" => TheRefusal::Variable,
"not" | "and" | "or" | "is" | "true" | "false" | "super" => TheRefusal::Syntax,
"in" | "contains" | "as" => TheRefusal::Future,
"exitapp" => TheRefusal::Func,
"class" => TheRefusal::Class,
_ => return None,
})
}
const OPERATORS: &[&str] = &[
">>>=", "<<=", "//=", ">>=", ":=", "+=", "-=", "*=", "/=", ".=", "|=", "^=", "&=", "~=", "!==",
"==", "!=", "<=", ">=", "&&", "||", "??", "**", "//", "<<", ">>>", ">>", "++", "--", "::",
"=>", "=", "<", ">", "+", "-", "*", "%", "/", ".", "(", ")", ",", "{", "}", "[", "]", "!", ":",
"?", "&", "|", "^", "~",
];
fn the_operators_of(dialect: &'static dyn crate::dialect::Dialect) -> Vec<&'static str> {
if !dialect.the_lexer_knows_the_legacy_operators() {
return OPERATORS.to_vec();
}
let mut operators = OPERATORS.to_vec();
let at = operators
.iter()
.position(|operator| *operator == "<")
.unwrap_or(0);
operators.insert(at, "<>");
operators
}
#[derive(Clone, Debug)]
pub struct SyntaxError {
pub message: String,
pub line: u32,
pub column: u32,
}
impl core::fmt::Display for SyntaxError {
fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
write!(
formatter,
"{} (line {}, column {})",
self.message, self.line, self.column
)
}
}
#[cfg(feature = "std")]
impl std::error::Error for SyntaxError {}
impl SyntaxError {
fn new(message: impl Into<String>, line: u32, column: u32) -> Self {
Self {
message: message.into(),
line,
column,
}
}
}
const THE_DIRECTIVES: [&str; 19] = [
"clipboardtimeout",
"dllload",
"errorstdout",
"hotif",
"hotiftimeout",
"hotstring",
"include",
"includeagain",
"inputlevel",
"maxthreads",
"maxthreadsbuffer",
"maxthreadsperhotkey",
"notrayicon",
"requires",
"singleinstance",
"suspendexempt",
"usehook",
"warn",
"winactivateforce",
];
pub fn carries_directive(name: &str) -> bool {
THE_DIRECTIVES.contains(&name.to_lowercase().as_str())
}
fn the_character_is_a_letter(character: char, marks: bool) -> bool {
character.is_alphabetic()
|| character == '_'
|| character == '\u{feff}'
|| (marks && matches!(character, '#' | '$' | '@'))
}
fn the_character_is_of_a_name(character: char, marks: bool) -> bool {
character.is_alphanumeric()
|| character == '_'
|| character == '\u{feff}'
|| (marks && matches!(character, '#' | '$' | '@'))
}
struct TheNameWithTheEscape {
name: String,
end: usize,
operator: Option<String>,
has_an_escape: bool,
}
fn the_name_with_the_escape(
chars: &[char],
start: usize,
marks: bool,
dialect: &'static dyn crate::dialect::Dialect,
) -> TheNameWithTheEscape {
let mut name = String::new();
let mut index = start;
let mut operator = None;
let mut has_an_escape = false;
while index < chars.len() {
let character = chars[index];
if the_character_is_of_a_name(character, marks) {
name.push(character);
index += 1;
continue;
}
if character != '`' || !dialect.the_escape_may_be_in_a_name() {
break;
}
has_an_escape = true;
let Some(next) = chars.get(index + 1).copied() else {
index += 1;
break;
};
index += 2;
let expanded = dialect.the_escape_of_the_dialect(next);
if expanded.is_whitespace() {
break;
}
if expanded == ',' {
break;
}
if OPERATORS.contains(&expanded.to_string().as_str()) {
operator = Some(expanded.to_string());
break;
}
name.push(expanded);
}
TheNameWithTheEscape {
name,
end: index,
operator,
has_an_escape,
}
}
fn the_name_of_a_message(name: &str) -> String {
format!("{:?}", name.replace('\u{feff}', "?"))
}
fn the_line_of_the_hash(chars: &[char], start: usize) -> String {
let mut end = start;
while end < chars.len() && chars[end] != '\n' {
end += 1;
}
chars[start..end]
.iter()
.collect::<String>()
.trim_end()
.to_string()
}
fn the_refusal_of_the_warn(
dialect: &dyn crate::dialect::Dialect,
chars: &[char],
hash_at: usize,
argument: &str,
line: u32,
column: u32,
) -> Result<(), SyntaxError> {
let (kind, mode) = crate::warnings::the_parts_of_the_argument(argument);
if !kind.is_empty() && dialect.the_type_of_the_warning(kind).is_none() {
let source = the_line_of_the_hash(chars, hash_at);
let text = source
.split(',')
.next()
.unwrap_or(source.as_str())
.trim_end();
return Err(SyntaxError::new(
format!("Parameter #1 invalid.\n Specifically: {text}"),
line,
column,
));
}
if !mode.is_empty() && dialect.the_mode_of_the_warning(mode).is_none() {
return Err(SyntaxError::new(
format!("Parameter #2 invalid.\n Specifically: {mode}"),
line,
column,
));
}
Ok(())
}
fn the_source_of_the_chars(chars: &[char], start: usize, end: usize) -> String {
let start = start.min(end).min(chars.len());
let end = end.min(chars.len());
let mut text: String = chars[start..end].iter().collect();
while text.ends_with([' ', '\t', '\r']) {
text.pop();
}
text
}
struct TheOptionsOfASection {
join: Option<String>,
ltrim: bool,
ltrim0: bool,
rtrim0: bool,
comments: bool,
accent: bool,
}
struct TheSection {
options: TheOptionsOfASection,
lines: Vec<String>,
rest: String,
end: usize,
rest_at: usize,
after: usize,
closed: bool,
}
fn the_section_at(
chars: &[char],
start: usize,
line: u32,
dialect: &'static dyn crate::dialect::Dialect,
) -> Result<Option<TheSection>, SyntaxError> {
let mut index = start;
while matches!(chars.get(index), Some(' ') | Some('\t')) {
index += 1;
}
if chars.get(index) != Some(&'(') {
return Ok(None);
}
let column = (index - start + 1) as u32;
index += 1;
let mut end_of_the_line = index;
while end_of_the_line < chars.len() && chars[end_of_the_line] != '\n' {
end_of_the_line += 1;
}
let mut text_of_the_options: String = chars[index..end_of_the_line].iter().collect();
if let Some(at) = text_of_the_options.find(';') {
text_of_the_options.truncate(at);
}
let mut options = TheOptionsOfASection {
join: None,
ltrim: false,
ltrim0: false,
rtrim0: false,
comments: false,
accent: false,
};
let mut the_join = String::new();
let mut the_join_is_there = false;
let mut an_option_is_not_known: Option<String> = None;
let mut a_paren_is_in_the_line = false;
for option in text_of_the_options.split(' ') {
if option.is_empty() {
continue;
}
let (name, argument) = match option.get(..4) {
Some(head) if head.eq_ignore_ascii_case("join") => ("join", Some(&option[4..])),
_ => (option, None),
};
let text = argument.unwrap_or(option);
if text.contains('(') || text.contains(')') {
a_paren_is_in_the_line = true;
}
match name.to_lowercase().as_str() {
"join" => {
the_join_is_there = true;
the_join = argument.unwrap_or("").to_string();
}
"ltrim" => options.ltrim = true,
"ltrim0" => options.ltrim0 = true,
"rtrim0" => options.rtrim0 = true,
"comments" | "comment" | "com" | "c" => options.comments = true,
"`" => options.accent = true,
_ => {
if an_option_is_not_known.is_none() {
an_option_is_not_known = Some(option.to_string());
}
}
}
}
if a_paren_is_in_the_line {
return Ok(None);
}
if let Some(option) = an_option_is_not_known {
return Err(SyntaxError::new(
format!("Invalid option.\n Specifically: {option}"),
line,
column,
));
}
if the_join_is_there {
options.join = Some(the_escapes_of_a_text(&the_join, options.accent, dialect));
}
let mut lines: Vec<String> = Vec::new();
let mut rest = String::new();
let mut rest_at = end_of_the_line;
let mut end = chars.len();
let mut closed = false;
let mut cursor = end_of_the_line;
while cursor < chars.len() {
let mut start_of_the_line = cursor + 1;
if chars.get(start_of_the_line) == Some(&'\r') {
start_of_the_line += 1;
}
let mut end_of_the_line = start_of_the_line;
while end_of_the_line < chars.len() && chars[end_of_the_line] != '\n' {
end_of_the_line += 1;
}
let text: String = chars[start_of_the_line..end_of_the_line].iter().collect();
let trimmed = text.trim_start_matches([' ', '\t']);
if let Some(after_the_parenthesis) = trimmed.strip_prefix(')') {
rest = after_the_parenthesis.to_string();
rest_at = start_of_the_line + (trimmed.len() - after_the_parenthesis.len());
end = end_of_the_line;
closed = true;
break;
}
lines.push(text);
cursor = end_of_the_line;
}
Ok(Some(TheSection {
options,
lines,
rest,
end,
rest_at,
after: end_of_the_line + 1,
closed,
}))
}
fn the_content_of_the_section(section: &TheSection) -> String {
let mut lines: Vec<String> = Vec::new();
for line in §ion.lines {
let mut text = line.clone();
if section.options.comments {
if let Some(at) = text.find(';') {
text.truncate(at);
}
}
if !section.options.rtrim0 {
while text.ends_with([' ', '\t', '\r']) {
text.pop();
}
}
lines.push(text);
}
let mut indentation = (0usize, ' ');
if !section.options.ltrim && !section.options.ltrim0 {
let first = lines.first().map(|line| line.as_str()).unwrap_or("");
let spaces = first.chars().take_while(|c| *c == ' ').count();
let tabs = first.chars().take_while(|c| *c == '\t').count();
indentation = if spaces > 0 {
(spaces, ' ')
} else {
(tabs, '\t')
};
}
if indentation.0 > 0 {
let (count, character) = indentation;
for line in lines.iter_mut() {
if line.chars().take(count).all(|c| c == character) {
*line = line.chars().skip(count).collect();
}
}
}
if section.options.ltrim {
for line in lines.iter_mut() {
*line = line.trim_start_matches([' ', '\t']).to_string();
}
}
let join = match §ion.options.join {
Some(join) => join.clone(),
None => "\n".to_string(),
};
let mut content = String::new();
for (number, line) in lines.iter().enumerate() {
if number > 0 {
content.push_str(&join);
}
content.push_str(line);
}
content
}
fn the_escapes_of_a_text(
text: &str,
accent: bool,
dialect: &'static dyn crate::dialect::Dialect,
) -> String {
if accent {
return text.to_string();
}
let mut result = String::new();
let mut characters = text.chars();
while let Some(character) = characters.next() {
if character == '`' {
match characters.next() {
Some(next) => result.push(dialect.the_escape_of_the_dialect(next)),
None => result.push('`'),
}
continue;
}
result.push(character);
}
result
}
fn the_tokens_of_the_content(content: &str) -> Result<Vec<Token>, SyntaxError> {
let mut tokens = tokenize(content)?;
if tokens
.last()
.map(|token| token.kind == TokenKind::Eof)
.unwrap_or(false)
{
tokens.pop();
}
let mut a_newline_was_there = false;
let mut the_tokens: Vec<Token> = Vec::new();
for mut token in tokens {
if token.kind == TokenKind::Newline {
a_newline_was_there = true;
continue;
}
if a_newline_was_there {
token.spaced = true;
a_newline_was_there = false;
}
the_tokens.push(token);
}
Ok(the_tokens)
}
fn the_code_of_a_line(text: &str) -> &str {
match text.find(';') {
Some(at) => &text[..at],
None => text,
}
}
fn the_last_character_is_a_name(text: &str) -> bool {
match the_code_of_a_line(text).trim_end().chars().last() {
Some(last) => the_character_is_of_a_name(last, false),
None => false,
}
}
fn the_line_the_section_joins(
chars: &[char],
the_starts_of_the_lines: &[usize],
the_line: u32,
) -> Option<(u32, usize, usize)> {
let mut number = the_line as usize;
while number > 1 {
number -= 1;
let start = the_starts_of_the_lines[number - 1];
let end = the_starts_of_the_lines
.get(number)
.copied()
.unwrap_or(chars.len());
let text: String = chars[start..end].iter().collect();
if !the_code_of_a_line(&text).trim().is_empty() {
return Some((number as u32, start, end));
}
}
None
}
pub fn tokenize(source: &str) -> Result<Vec<Token>, SyntaxError> {
tokenize_with(source, crate::dialect::v2())
}
pub fn tokenize_with(
source: &str,
dialect: &'static dyn crate::dialect::Dialect,
) -> Result<Vec<Token>, SyntaxError> {
let source = source.strip_prefix('\u{feff}').unwrap_or(source);
let chars: Vec<char> = source.chars().collect();
let marks = dialect.accepts_a_hash_a_dollar_and_an_at_in_a_name();
let operators = the_operators_of(dialect);
let mut tokens: Vec<Token> = Vec::new();
let mut index = 0usize;
let mut line = 1u32;
let mut column = 1u32;
let mut spaced = false;
let mut the_starts_of_the_lines: Vec<usize> = vec![0];
while index < chars.len() {
let current = chars[index];
let before = spaced;
spaced = false;
if current == '\n' {
if let Some(section) = the_section_at(&chars, index + 1, line + 1, dialect)? {
let mut cursor = index + 1;
while cursor < section.end {
the_starts_of_the_lines.push(cursor);
match chars[cursor..section.end].iter().position(|c| *c == '\n') {
Some(at) => cursor += at + 1,
None => break,
}
}
the_section_joins_the_line_above(
&mut tokens,
&chars,
§ion,
&the_starts_of_the_lines,
line,
)?;
index = section.end;
line += section.lines.len() as u32 + 2;
column = 1;
continue;
}
tokens.push(Token {
kind: TokenKind::Newline,
value: "\n".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
line += 1;
column = 1;
the_starts_of_the_lines.push(index);
continue;
}
if current == ' ' || current == '\t' || current == '\r' {
index += 1;
column += 1;
spaced = true;
continue;
}
if current == '#' {
if tokens.iter().any(|token| token.line == line) {
tokens.push(Token {
kind: TokenKind::Op,
value: "#".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
column += 1;
continue;
}
let (start_line, start_column) = (line, column);
let hash_at = index;
index += 1;
column += 1;
let mut name = String::new();
while index < chars.len() && the_character_is_of_a_name(chars[index], marks) {
name.push(chars[index]);
index += 1;
column += 1;
}
if name.is_empty() || name.starts_with(|character: char| character.is_ascii_digit()) {
return Err(SyntaxError::new(
format!(
"This line does not contain a recognized action.\n Specifically: {}",
the_line_of_the_hash(&chars, hash_at)
),
start_line,
start_column,
));
}
let mut argument = String::new();
while index < chars.len() && chars[index] != '\n' {
argument.push(chars[index]);
index += 1;
column += 1;
}
if !dialect.carries_directive(&name) {
return Err(SyntaxError::new(
format!(
"This line does not contain a recognized action.\n Specifically: #{name}{}",
argument.trim_end()
),
start_line,
start_column,
));
}
if name.eq_ignore_ascii_case("warn") {
the_refusal_of_the_warn(
dialect,
&chars,
hash_at,
&argument,
start_line,
start_column,
)?;
}
tokens.push(Token {
kind: TokenKind::Directive,
value: name.to_lowercase(),
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
let argument = argument.trim().to_string();
if !argument.is_empty() {
tokens.push(Token {
kind: TokenKind::String,
value: argument,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
}
continue;
}
if current == ';' {
if before || !tokens.iter().any(|token| token.line == line) {
while index < chars.len() && chars[index] != '\n' {
index += 1;
}
continue;
}
tokens.push(Token {
kind: TokenKind::Op,
value: ";".into(),
line,
column,
spaced: before,
quote: None,
});
index += 1;
column += 1;
continue;
}
if current == '\'' && dialect.the_single_quote_is_not_a_text() {
let (start_line, start_column) = (line, column);
let mut end = index;
while end < chars.len() && !the_character_ends_the_symbol_of_the_v1(chars[end]) {
end += 1;
}
let symbol: String = chars[index..end].iter().collect();
column += (end - index) as u32;
index = end;
tokens.push(Token {
kind: TokenKind::Name,
value: symbol,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
if current == '"' || current == '\'' {
let (start_line, start_column) = (line, column);
let quote = current;
let quote_index = index;
index += 1;
column += 1;
let mut buffer = String::new();
let mut the_text_of_the_section = String::new();
loop {
while index < chars.len() && chars[index] != '\n' {
if chars[index] == '"'
&& dialect.the_quote_escapes_the_quote()
&& index + 1 < chars.len()
&& chars[index + 1] == '"'
{
buffer.push('"');
index += 2;
column += 2;
continue;
}
if chars[index] == quote {
break;
}
if chars[index] == '`' && index + 1 < chars.len() {
let next = chars[index + 1];
if next == quote && !dialect.the_escape_protects_the_quote_of_a_text() {
index += 1;
column += 1;
break;
}
buffer.push(dialect.the_escape_of_the_dialect(next));
index += 2;
column += 2;
continue;
}
buffer.push(chars[index]);
index += 1;
column += 1;
}
if index >= chars.len() || chars[index] != '\n' {
break;
}
let Some(section) = the_section_at(&chars, index + 1, line + 1, dialect)? else {
break;
};
if !section.closed {
let start_of_the_line = quote_index - (start_column as usize - 1);
let mut text: String = chars[start_of_the_line..index].iter().collect();
let rest: String = chars[section.after..].iter().collect();
text.push_str(&rest);
return Err(SyntaxError::new(
format!("Missing \")\"\n Specifically: {text}"),
start_line,
start_column,
));
}
let content = the_content_of_the_section(§ion);
the_text_of_the_section = content.clone();
buffer.push_str(&the_escapes_of_a_text(
&content,
section.options.accent,
dialect,
));
index = section.rest_at;
line += section.lines.len() as u32 + 2;
column = 1;
}
if index >= chars.len() || chars[index] == '\n' {
let start = quote_index - (start_column as usize - 1);
let mut end = index;
while end < chars.len() && chars[end] != '\n' {
end += 1;
}
let mut text = the_source_of_the_chars(&chars, start, end);
if !the_text_of_the_section.is_empty() {
let mut end_of_the_line = start;
while end_of_the_line < chars.len() && chars[end_of_the_line] != '\n' {
end_of_the_line += 1;
}
text = the_source_of_the_chars(&chars, start, end_of_the_line);
text.push_str(&the_text_of_the_section);
}
if dialect.the_text_that_is_not_closed_is_named_alone() {
let mut end = index;
while end < chars.len() && chars[end] != '\n' {
end += 1;
}
let text = the_source_of_the_chars(&chars, quote_index, end);
if dialect.the_quote_that_is_not_closed_is_an_error() {
return Err(SyntaxError::new(
format!("Missing close-quote\n Specifically: {text}"),
start_line,
start_column,
));
}
tokens.push(Token {
kind: TokenKind::UnclosedText,
value: text,
line: start_line,
column: start_column,
spaced: before,
quote: Some(quote),
});
continue;
}
return Err(SyntaxError::new(
format!("Missing \"{quote}\"\n Specifically: {text}"),
start_line,
start_column,
));
}
index += 1;
column += 1;
tokens.push(Token {
kind: TokenKind::String,
value: buffer,
line: start_line,
column: start_column,
spaced: before,
quote: Some(quote),
});
continue;
}
if current.is_ascii_digit()
|| (current == '.' && the_point_begins_a_number(&chars, index, &tokens))
{
let (start_line, start_column) = (line, column);
let start = index;
index =
the_end_of_the_number(&chars, index, dialect.the_exponent_needs_a_point(), marks);
column += (index - start) as u32;
if index < chars.len()
&& (the_character_is_a_letter(chars[index], marks)
|| (chars[index] == '`' && dialect.the_escape_may_be_in_a_name()))
{
let number_end = index;
let mut name = chars[start..index].iter().collect::<String>();
if chars[index] == '`' {
let escape = the_name_with_the_escape(&chars, index, marks, dialect);
if let Some(operator) = escape.operator {
column += (escape.end - number_end) as u32;
index = escape.end;
tokens.push(Token {
kind: TokenKind::Number,
value: chars[start..number_end].iter().collect(),
quote: None,
line: start_line,
column: start_column,
spaced: before,
});
tokens.push(Token {
kind: TokenKind::Op,
value: operator,
quote: None,
line: start_line,
column: start_column + (number_end - start) as u32,
spaced: false,
});
continue;
}
name.push_str(&escape.name);
index = escape.end;
}
while index < chars.len() && the_character_is_of_a_name(chars[index], marks) {
name.push(chars[index]);
index += 1;
}
if chars.get(index) == Some(&':') {
column += (index - number_end) as u32;
tokens.push(Token {
kind: TokenKind::Name,
value: name,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
if dialect.the_number_may_be_followed_by_a_name() {
column += (index - number_end) as u32;
tokens.push(Token {
kind: TokenKind::Name,
value: name,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
return Err(SyntaxError::new(
format!(
"This variable name starts with a number, which is not allowed:\n{}",
the_name_of_a_message(&name)
),
start_line,
start_column,
));
}
if dialect.the_name_may_not_carry_an_illegal_character()
&& chars.get(index) == Some(&'"')
{
let text: String = chars[start..index].iter().collect();
if text.contains('.') {
return Err(SyntaxError::new(
format!(
"This character is not allowed here.\n Specifically: {}",
the_source_of_a_text(&chars, index)
),
start_line,
start_column,
));
}
let end = the_end_of_a_symbol_that_carries_a_quote(&chars, index);
let word: String = chars[start..end].iter().collect();
column += (end - index) as u32;
index = end;
tokens.push(Token {
kind: TokenKind::Name,
value: word,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
tokens.push(Token {
kind: TokenKind::Number,
value: chars[start..index].iter().collect(),
quote: None,
line: start_line,
column: start_column,
spaced: before,
});
continue;
}
if the_character_is_a_letter(current, marks)
|| (current == '`' && dialect.the_escape_may_be_in_a_name())
{
let (start_line, start_column) = (line, column);
let start = index;
let name = the_name_with_the_escape(&chars, index, marks, dialect);
index = name.end;
column += (index - start) as u32;
if !name.has_an_escape
&& dialect.the_name_may_not_carry_an_illegal_character()
&& chars.get(index) == Some(&'"')
{
let end = the_end_of_a_symbol_that_carries_a_quote(&chars, index);
let word: String = chars[start..end].iter().collect();
column += (end - index) as u32;
index = end;
tokens.push(Token {
kind: TokenKind::Name,
value: word,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
continue;
}
let word = name.name;
let lower = word.to_lowercase();
let kind = if KEYWORDS.contains(&lower.as_str())
&& !(dialect.the_words_may_be_the_name_of_a_variable()
&& matches!(lower.as_str(), "is" | "as" | "in" | "contains" | "super"))
{
TokenKind::Keyword
} else {
TokenKind::Name
};
tokens.push(Token {
kind,
value: word,
line: start_line,
column: start_column,
spaced: before,
quote: None,
});
if let Some(operator) = name.operator {
tokens.push(Token {
kind: TokenKind::Op,
value: operator,
quote: None,
line: start_line,
column: start_column,
spaced: false,
});
}
continue;
}
if current == '\\' && dialect.accepts_a_bare_backslash() {
tokens.push(Token {
kind: TokenKind::Name,
value: "\\".into(),
quote: None,
line,
column,
spaced: before,
});
index += 1;
column += 1;
continue;
}
let mut matched = false;
for operator in operators.iter().copied() {
if matches!(operator, "++" | "--")
&& !dialect.the_increment_after_a_number_is_an_operator()
&& tokens
.last()
.map(|token| token.kind == TokenKind::Number)
.unwrap_or(false)
{
continue;
}
if chars[index..].starts_with(&operator.chars().collect::<Vec<char>>()[..]) {
tokens.push(Token {
kind: TokenKind::Op,
value: (*operator).to_string(),
quote: None,
line,
column,
spaced: before,
});
index += operator.chars().count();
column += operator.chars().count() as u32;
matched = true;
break;
}
}
if !matched {
return Err(SyntaxError::new(
format!("unexpected character {current:?}"),
line,
column,
));
}
}
tokens.push(Token {
kind: TokenKind::Eof,
value: String::new(),
line,
column,
spaced: false,
quote: None,
});
Ok(tokens)
}
fn the_section_joins_the_line_above(
tokens: &mut Vec<Token>,
chars: &[char],
section: &TheSection,
the_starts_of_the_lines: &[usize],
line: u32,
) -> Result<(), SyntaxError> {
while tokens
.last()
.map(|token| token.kind == TokenKind::Newline)
.unwrap_or(false)
{
tokens.pop();
}
let (the_number, the_start, the_end) =
match the_line_the_section_joins(chars, the_starts_of_the_lines, line + 1) {
Some(above) => above,
None => (line + 1, line as usize, line as usize),
};
let text_of_the_line: String = chars[the_start..the_end].iter().collect();
let a_space = the_number != line + 1 && the_last_character_is_a_name(&text_of_the_line);
let content = the_content_of_the_section(section);
if !section.closed {
let mut text = text_of_the_line.trim_end().to_string();
if a_space {
text.push(' ');
}
let mut rest: String = chars[section.after..].iter().collect();
if rest.ends_with("\r\n") {
rest.truncate(rest.len() - 2);
} else if rest.ends_with('\n') {
rest.pop();
}
text.push_str(&rest);
return Err(SyntaxError::new(
format!("Missing \")\"\n Specifically: {text}"),
the_number,
1,
));
}
let the_directive_is_the_line_above = tokens
.iter()
.rev()
.take_while(|token| token.line == the_number)
.any(|token| token.kind == TokenKind::Directive);
if the_directive_is_the_line_above {
let mut argument = String::new();
if tokens
.last()
.map(|token| token.kind == TokenKind::String)
.unwrap_or(false)
{
argument = tokens.pop().map(|token| token.value).unwrap_or_default();
}
if a_space {
argument.push(' ');
}
argument.push_str(&content);
argument.push_str(§ion.rest);
tokens.push(Token {
kind: TokenKind::String,
value: argument,
line: the_number,
column: 1,
spaced: false,
quote: None,
});
return Ok(());
}
let mut the_tokens = the_tokens_of_the_content(&content).map_err(|mut error| {
error.line = the_number;
error
})?;
for token in the_tokens.iter_mut() {
token.line = the_number;
}
if let Some(first) = the_tokens.first_mut() {
first.spaced = first.spaced || a_space;
}
tokens.extend(the_tokens);
if !section.rest.is_empty() {
let mut the_tokens = the_tokens_of_the_content(§ion.rest).map_err(|mut error| {
error.line = the_number;
error
})?;
for token in the_tokens.iter_mut() {
token.line = the_number;
}
tokens.extend(the_tokens);
}
Ok(())
}
fn the_point_begins_a_number(chars: &[char], index: usize, tokens: &[Token]) -> bool {
if !chars
.get(index + 1)
.is_some_and(|next| next.is_ascii_digit())
{
return false;
}
match tokens.last() {
Some(token) => !the_token_ends_a_value(token),
None => true,
}
}
fn the_token_ends_a_value(token: &Token) -> bool {
match token.kind {
TokenKind::Name | TokenKind::Number | TokenKind::String => true,
TokenKind::Op => matches!(token.value.as_str(), ")" | "]" | "}"),
_ => false,
}
}
fn the_end_of_the_number(
chars: &[char],
start: usize,
the_exponent_needs_a_point: bool,
marks: bool,
) -> usize {
if chars.get(start) == Some(&'0')
&& matches!(chars.get(start + 1), Some(&'x') | Some(&'X'))
&& chars
.get(start + 2)
.is_some_and(|digit| digit.is_ascii_hexdigit())
{
let mut index = start + 2;
while index < chars.len() && chars[index].is_ascii_hexdigit() {
index += 1;
}
return index;
}
let mut index = start;
while index < chars.len() && chars[index].is_ascii_digit() {
index += 1;
}
let mut the_number_has_a_point = false;
if chars.get(index) == Some(&'.') {
if !chars
.get(index + 1)
.is_some_and(|character| the_character_is_a_letter(*character, marks))
{
the_number_has_a_point = true;
index += 1;
while index < chars.len() && chars[index].is_ascii_digit() {
index += 1;
}
}
}
if (!the_exponent_needs_a_point || the_number_has_a_point)
&& matches!(chars.get(index), Some(&'e') | Some(&'E'))
{
let mut after = index + 1;
if matches!(chars.get(after), Some(&'+') | Some(&'-')) {
after += 1;
}
if chars.get(after).is_some_and(|digit| digit.is_ascii_digit()) {
while after < chars.len() && chars[after].is_ascii_digit() {
after += 1;
}
return after;
}
}
index
}
#[cfg(test)]
mod tests {
use super::*;
fn kinds(source: &str) -> Vec<TokenKind> {
tokenize(source)
.expect("parses")
.iter()
.map(|t| t.kind)
.collect()
}
#[test]
fn a_string_is_one_token() {
let tokens = tokenize("Send \"hello world\"").expect("parses");
assert_eq!(tokens[0].value, "Send");
assert_eq!(tokens[1].kind, TokenKind::String);
assert_eq!(tokens[1].value, "hello world");
}
#[test]
fn a_comment_is_not_a_token() {
assert_eq!(
kinds("Send \"a\" ; this is a comment"),
vec![TokenKind::Name, TokenKind::String, TokenKind::Eof]
);
}
#[test]
fn an_unterminated_string_is_an_error() {
let error = tokenize("Send \"never closed").expect_err("fails");
assert_eq!(error.line, 1);
assert_eq!(
error.message,
"Missing \"\"\"\n Specifically: Send \"never closed"
);
let error = tokenize("Send 'never closed").expect_err("fails");
assert_eq!(error.line, 1);
assert_eq!(
error.message,
"Missing \"'\"\n Specifically: Send 'never closed"
);
}
#[test]
fn a_text_is_between_double_quotes_or_between_single_quotes() {
let tokens = tokenize("x := 'hello'").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::String);
assert_eq!(tokens[2].value, "hello");
assert_eq!(tokens[2].quote, Some('\''));
let tokens = tokenize("x := \"hello\"").expect("parses");
assert_eq!(tokens[2].quote, Some('"'));
}
#[test]
fn the_escape_of_the_dialect_is_the_backtick_in_the_two_kinds_of_text() {
let tokens = tokenize("x := 'a`nb'").expect("parses");
assert_eq!(tokens[2].value, "a\nb");
let tokens = tokenize("x := \"a`nb\"").expect("parses");
assert_eq!(tokens[2].value, "a\nb");
}
#[test]
fn a_text_does_not_cross_a_line() {
let error = tokenize("x := \"a\nb\"").expect_err("fails");
assert_eq!(error.line, 1);
assert_eq!(error.message, "Missing \"\"\"\n Specifically: x := \"a");
}
#[test]
fn a_number_keeps_its_point() {
let tokens = tokenize("x := 3.5").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "3.5");
}
#[test]
fn the_hex_of_v2_is_one_token() {
let tokens = tokenize("x := 0xE9").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "0xE9");
}
#[test]
fn the_exponent_of_v2_is_one_token() {
let tokens = tokenize("x := 1e3").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "1e3");
}
#[test]
fn a_number_has_one_point() {
let tokens = tokenize("x := 1.5.2").expect("parses");
assert_eq!(tokens[2].value, "1.5");
assert_eq!(tokens[3].value, ".");
}
#[test]
fn a_name_that_begins_with_a_number_is_an_error() {
let error = tokenize("x := 12abc").expect_err("fails");
assert_eq!(
error.message,
"This variable name starts with a number, which is not allowed:\n\"12abc\""
);
assert_eq!(error.line, 1);
}
#[test]
fn the_longest_operator_wins() {
let tokens = tokenize("x := a >>>= b").expect("parses");
assert_eq!(tokens[3].value, ">>>=");
}
#[test]
fn the_assignment_operators_of_v2_are_one_token() {
for operator in ["|=", "^=", "&=", "~="] {
let tokens = tokenize(&format!("x {operator} 2")).expect("parses");
assert_eq!(tokens[1].value, operator);
}
}
#[test]
fn a_decrement_after_a_number_is_a_subtraction() {
let tokens = tokenize("x := 5--3").expect("parses");
assert_eq!(tokens[3].value, "-");
}
#[test]
fn a_decrement_after_a_name_is_a_decrement() {
let tokens = tokenize("x--").expect("parses");
assert_eq!(tokens[1].value, "--");
}
#[test]
fn a_directive_is_a_directive() {
let tokens = tokenize("#Requires AutoHotkey v2.0.19\n").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Directive);
assert_eq!(tokens[0].value, "requires");
assert_eq!(tokens[1].kind, TokenKind::String);
assert_eq!(tokens[1].value, "AutoHotkey v2.0.19");
}
#[test]
fn the_word_for_is_a_keyword() {
let tokens = tokenize("for x in y").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Keyword);
assert_eq!(tokens[0].value, "for");
assert_eq!(tokens[2].kind, TokenKind::Keyword);
assert_eq!(tokens[2].value, "in");
}
#[test]
fn the_word_operators_are_keywords() {
let tokens = tokenize("a is b and c").expect("parses");
assert_eq!(tokens[1].kind, TokenKind::Keyword);
assert_eq!(tokens[1].value, "is");
assert_eq!(tokens[3].kind, TokenKind::Keyword);
assert_eq!(tokens[3].value, "and");
}
#[test]
fn the_script_may_begin_with_the_byte_order_mark() {
let tokens = tokenize("\u{feff}x := 1\n").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Name);
assert_eq!(tokens[0].value, "x");
}
#[test]
fn the_byte_order_mark_is_a_letter_of_a_name() {
let tokens = tokenize("x\u{feff}y := 1\n").expect("parses");
assert_eq!(tokens[0].kind, TokenKind::Name);
assert_eq!(tokens[0].value, "x\u{feff}y");
}
#[test]
fn the_name_that_begins_with_a_number_and_a_mark_is_refused() {
let error = tokenize("1\u{feff}x := 1\n").expect_err("does not lex");
assert_eq!(
error.message,
"This variable name starts with a number, which is not allowed:\n\"1?x\""
);
}
#[test]
fn the_directive_without_a_name_is_a_line_that_holds_no_action() {
let error = tokenize("#\n").expect_err("does not lex");
assert_eq!(
error.message,
"This line does not contain a recognized action.\n Specifically: #"
);
}
#[test]
fn the_directive_with_a_number_is_a_line_that_holds_no_action() {
let error = tokenize("#123\n").expect_err("does not lex");
assert_eq!(
error.message,
"This line does not contain a recognized action.\n Specifically: #123"
);
}
#[test]
fn the_directive_that_is_not_known_is_a_line_that_holds_no_action() {
let error = tokenize("#Foo Bar\n").expect_err("does not lex");
assert_eq!(
error.message,
"This line does not contain a recognized action.\n Specifically: #Foo Bar"
);
}
#[test]
fn the_semicolon_is_a_comment_only_when_it_is_apart() {
let tokens = tokenize("x := 1;c").expect("parses");
assert_eq!(tokens[3].kind, TokenKind::Op);
assert_eq!(tokens[3].value, ";");
let tokens = tokenize("x := 1 ;c").expect("parses");
assert_eq!(tokens.len(), 4);
assert_eq!(tokens[3].kind, TokenKind::Eof);
let tokens = tokenize(";c").expect("parses");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0].kind, TokenKind::Eof);
}
#[test]
fn the_point_of_a_number_is_not_the_point_of_a_member() {
let tokens = tokenize("x := 1.ToString").expect("parses");
assert_eq!(tokens[2].kind, TokenKind::Number);
assert_eq!(tokens[2].value, "1");
assert_eq!(tokens[3].value, ".");
assert_eq!(tokens[4].value, "ToString");
let tokens = tokenize("x := 1.").expect("parses");
assert_eq!(tokens[2].value, "1.");
}
#[test]
fn the_three_way_comparison_is_not_a_token() {
let tokens = tokenize("x := 1 === 1").expect("parses");
assert_eq!(tokens[3].value, "==");
assert_eq!(tokens[4].value, "=");
}
}