use std::{collections::HashMap, sync::Arc};
use super::{CodeToken, LineState, ScanState, TokenName, Tokenizer, token};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum CodeMode {
#[default]
Code,
Markup,
Markdown,
}
#[derive(Clone, Debug, PartialEq)]
pub struct BlockComment {
pub open: Arc<str>,
pub close: Arc<str>,
pub nests: bool,
pub name: TokenName,
}
#[derive(Clone, Debug, PartialEq)]
pub struct StringForm {
pub open: Arc<str>,
pub close: Arc<str>,
pub prefixes: Vec<Arc<str>>,
pub escapes: bool,
pub doubled: bool,
pub multiline: bool,
pub hashes: bool,
pub name: TokenName,
}
impl StringForm {
pub fn new(open: &str, close: &str) -> Self {
Self {
open: Arc::from(open),
close: Arc::from(close),
prefixes: vec![Arc::from("")],
escapes: true,
doubled: false,
multiline: false,
hashes: false,
name: token::STRING.into(),
}
}
pub fn prefixes(mut self, prefixes: &[&str]) -> Self {
self.prefixes = prefixes.iter().map(|prefix| Arc::from(*prefix)).collect();
self
}
pub fn escapes(mut self, escapes: bool) -> Self {
self.escapes = escapes;
self
}
pub fn doubled(mut self, doubled: bool) -> Self {
self.doubled = doubled;
self
}
pub fn multiline(mut self, multiline: bool) -> Self {
self.multiline = multiline;
self
}
pub fn hashes(mut self, hashes: bool) -> Self {
self.hashes = hashes;
self
}
pub fn name(mut self, name: impl Into<TokenName>) -> Self {
self.name = name.into();
self
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct NumberForms {
pub prefixed: bool,
pub underscores: bool,
pub suffixes: bool,
pub dates: bool,
}
impl Default for NumberForms {
fn default() -> Self {
Self {
prefixed: true,
underscores: false,
suffixes: false,
dates: false,
}
}
}
#[derive(Clone, Debug, Default, PartialEq)]
pub enum KeyRule {
#[default]
None,
Colon,
Equals,
Declaration,
}
#[derive(Clone, Debug)]
pub struct CodeLanguage {
pub(super) name: Arc<str>,
pub(super) aliases: Vec<Arc<str>>,
pub(super) mode: CodeMode,
pub(super) words: HashMap<Box<str>, TokenName>,
pub(super) any_case: bool,
pub(super) line_comments: Vec<(Arc<str>, TokenName)>,
pub(super) comment_after_space: bool,
pub(super) block_comments: Vec<BlockComment>,
pub(super) strings: Vec<StringForm>,
pub(super) prefixes_any_case: bool,
pub(super) characters: bool,
pub(super) numbers: NumberForms,
pub(super) capitals_are_types: bool,
pub(super) path: Option<Arc<str>>,
pub(super) calls: bool,
pub(super) macros: bool,
pub(super) declares: HashMap<Box<str>, TokenName>,
pub(super) attributes: Vec<Arc<str>>,
pub(super) at_words: bool,
pub(super) variables: bool,
pub(super) name_chars: Arc<str>,
pub(super) name_starts: Arc<str>,
pub(super) keys: KeyRule,
pub(super) sections: bool,
pub(super) regexes: bool,
pub(super) commands: bool,
pub(super) commands_after: Vec<Box<str>>,
pub(super) selectors: bool,
pub(super) block_scalars: bool,
pub(super) operators: Arc<str>,
}
impl CodeLanguage {
pub fn new(name: &str) -> Self {
Self {
name: Arc::from(name),
aliases: Vec::new(),
mode: CodeMode::Code,
words: HashMap::new(),
any_case: false,
line_comments: Vec::new(),
comment_after_space: false,
block_comments: Vec::new(),
strings: Vec::new(),
prefixes_any_case: false,
characters: false,
numbers: NumberForms::default(),
capitals_are_types: false,
path: None,
calls: false,
macros: false,
declares: HashMap::new(),
attributes: Vec::new(),
at_words: false,
variables: false,
name_chars: Arc::from("_"),
name_starts: Arc::from("_"),
keys: KeyRule::None,
sections: false,
regexes: false,
commands: false,
commands_after: Vec::new(),
selectors: false,
block_scalars: false,
operators: Arc::from("=+-*/%<>!&|^~?:"),
}
}
pub fn name(&self) -> Arc<str> {
self.name.clone()
}
pub fn aliases_of(&self) -> &[Arc<str>] {
&self.aliases
}
pub fn aliases(mut self, aliases: &[&str]) -> Self {
self.aliases.extend(aliases.iter().map(|alias| Arc::from(*alias)));
self
}
pub fn mode(mut self, mode: CodeMode) -> Self {
self.mode = mode;
self
}
pub fn words(mut self, name: impl Into<TokenName>, words: &[&str]) -> Self {
let name = name.into();
for word in words {
let word = if self.any_case {
word.to_lowercase()
} else {
(*word).to_owned()
};
self.words.insert(word.into_boxed_str(), name);
}
self
}
pub fn words_any_case(mut self, any_case: bool) -> Self {
self.any_case = any_case;
self
}
pub fn line_comments(mut self, marks: &[&str]) -> Self {
for mark in marks {
self.line_comments.push((Arc::from(*mark), token::COMMENT.into()));
}
self
.line_comments
.sort_by_key(|(mark, _)| std::cmp::Reverse(mark.len()));
self
}
pub fn doc_comments(mut self, marks: &[&str]) -> Self {
for mark in marks {
self.line_comments.push((Arc::from(*mark), token::COMMENT_DOC.into()));
}
self
.line_comments
.sort_by_key(|(mark, _)| std::cmp::Reverse(mark.len()));
self
}
pub fn comment_after_space(mut self, after_space: bool) -> Self {
self.comment_after_space = after_space;
self
}
pub fn block_comment(mut self, open: &str, close: &str, nests: bool) -> Self {
self.block_comments.push(BlockComment {
open: Arc::from(open),
close: Arc::from(close),
nests,
name: token::COMMENT.into(),
});
self
.block_comments
.sort_by_key(|comment| std::cmp::Reverse(comment.open.len()));
self
}
pub fn doc_block_comment(mut self, open: &str, close: &str) -> Self {
self.block_comments.push(BlockComment {
open: Arc::from(open),
close: Arc::from(close),
nests: false,
name: token::COMMENT_DOC.into(),
});
self
.block_comments
.sort_by_key(|comment| std::cmp::Reverse(comment.open.len()));
self
}
pub fn strings(mut self, forms: Vec<StringForm>) -> Self {
self.strings.extend(forms);
self
}
pub fn prefixes_any_case(mut self, any_case: bool) -> Self {
self.prefixes_any_case = any_case;
self
}
pub fn characters(mut self, characters: bool) -> Self {
self.characters = characters;
self
}
pub fn numbers(mut self, numbers: NumberForms) -> Self {
self.numbers = numbers;
self
}
pub fn capitals_are_types(mut self, capitals: bool) -> Self {
self.capitals_are_types = capitals;
self
}
pub fn path(mut self, separator: &str) -> Self {
self.path = Some(Arc::from(separator));
self
}
pub fn calls(mut self, calls: bool) -> Self {
self.calls = calls;
self
}
pub fn macros(mut self, macros: bool) -> Self {
self.macros = macros;
self
}
pub fn declares(mut self, name: impl Into<TokenName>, words: &[&str]) -> Self {
let name = name.into();
for word in words {
self.declares.insert((*word).into(), name);
}
self
}
pub fn attributes(mut self, marks: &[&str]) -> Self {
self.attributes.extend(marks.iter().map(|mark| Arc::from(*mark)));
self.attributes.sort_by_key(|mark| std::cmp::Reverse(mark.len()));
self
}
pub fn at_words(mut self, at_words: bool) -> Self {
self.at_words = at_words;
self
}
pub fn variables(mut self, variables: bool) -> Self {
self.variables = variables;
self
}
pub fn name_chars(mut self, inside: &str, start: &str) -> Self {
self.name_chars = Arc::from(inside);
self.name_starts = Arc::from(start);
self
}
pub fn keys(mut self, keys: KeyRule) -> Self {
self.keys = keys;
self
}
pub fn sections(mut self, sections: bool) -> Self {
self.sections = sections;
self
}
pub fn regexes(mut self, regexes: bool) -> Self {
self.regexes = regexes;
self
}
pub fn commands(mut self, commands: bool) -> Self {
self.commands = commands;
self
}
pub fn commands_after(mut self, words: &[&str]) -> Self {
self.commands_after.extend(words.iter().map(|word| Box::from(*word)));
self
}
pub fn selectors(mut self, selectors: bool) -> Self {
self.selectors = selectors;
self
}
pub fn block_scalars(mut self, block_scalars: bool) -> Self {
self.block_scalars = block_scalars;
self
}
pub fn operators(mut self, operators: &str) -> Self {
self.operators = Arc::from(operators);
self
}
pub fn tokens(&self, code: &str) -> Vec<CodeToken> {
let mut tokens = Vec::new();
let mut state = ScanState::default();
let mut start = 0;
for line in code.split('\n') {
self.scan_line(line, start, &mut state, &mut tokens);
start += line.len() + 1;
}
tokens
}
pub fn line_tokens(&self, line: &str, state: &mut ScanState) -> Vec<CodeToken> {
let mut tokens = Vec::new();
self.scan_line(line.trim_end_matches('\n'), 0, state, &mut tokens);
tokens
}
pub fn states(&self, code: &str) -> Vec<ScanState> {
let mut states = Vec::new();
let mut state = ScanState::default();
let mut scratch = Vec::new();
for line in code.split('\n') {
states.push(state.clone());
scratch.clear();
self.scan_line(line, 0, &mut state, &mut scratch);
}
states
}
}
impl Tokenizer for CodeLanguage {
fn tokens(&self, code: &str) -> Vec<CodeToken> {
CodeLanguage::tokens(self, code)
}
fn tokens_from(&self, lines: &[&str], start: &LineState) -> Option<Vec<(Vec<CodeToken>, LineState)>> {
let mut state = start.get::<ScanState>().cloned().unwrap_or_default();
Some(
lines
.iter()
.map(|line| {
let tokens = self.line_tokens(line, &mut state);
(tokens, LineState::new(state.clone()))
})
.collect(),
)
}
}