#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct LineRule {
pub marker: &'static str,
pub anchored: bool,
pub ci: bool,
pub not_after: &'static [u8],
pub boundary_before: bool,
pub require_after: Option<&'static [u8]>,
}
impl LineRule {
const fn new(marker: &'static str) -> Self {
LineRule {
marker,
anchored: false,
ci: false,
not_after: &[],
boundary_before: false,
require_after: None,
}
}
const fn anchored(mut self) -> Self {
self.anchored = true;
self
}
const fn word(mut self) -> Self {
self.ci = true;
self.require_after = Some(b" \t\r");
self
}
const fn not_after(mut self, bytes: &'static [u8]) -> Self {
self.not_after = bytes;
self
}
const fn boundary_before(mut self) -> Self {
self.boundary_before = true;
self
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BlockRule {
pub open: &'static str,
pub close: &'static str,
pub nestable: bool,
pub anchored: bool,
pub close_anchored: bool,
}
impl BlockRule {
const fn new(open: &'static str, close: &'static str) -> Self {
BlockRule {
open,
close,
nestable: false,
anchored: false,
close_anchored: false,
}
}
const fn nestable(mut self) -> Self {
self.nestable = true;
self
}
const fn line_anchored(mut self) -> Self {
self.anchored = true;
self.close_anchored = true;
self
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct StringRule {
pub open: &'static str,
pub close: &'static str,
pub escape: Option<u8>,
pub multiline: bool,
pub doubling: bool,
pub raw_prefix: bool,
pub char_like: bool,
pub doc_capable: bool,
}
impl StringRule {
const fn new(open: &'static str, close: &'static str) -> Self {
StringRule {
open,
close,
escape: Some(b'\\'),
multiline: false,
doubling: false,
raw_prefix: false,
char_like: false,
doc_capable: false,
}
}
const fn quote(q: &'static str) -> Self {
Self::new(q, q)
}
const fn no_escape(mut self) -> Self {
self.escape = None;
self
}
const fn multiline(mut self) -> Self {
self.multiline = true;
self
}
const fn doubling(mut self) -> Self {
self.doubling = true;
self
}
const fn raw_prefix(mut self) -> Self {
self.raw_prefix = true;
self
}
const fn char_like(mut self) -> Self {
self.char_like = true;
self
}
const fn doc_capable(mut self) -> Self {
self.doc_capable = true;
self
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Dialect {
Unknown,
CFamily,
Hash,
Sql,
Lua,
Haskell,
Lisp,
Markup,
Ml,
Percent,
VisualBasic,
Batch,
Fortran,
Ini,
}
impl Dialect {
pub const ALL: &'static [Dialect] = &[
Dialect::CFamily,
Dialect::Hash,
Dialect::Sql,
Dialect::Lua,
Dialect::Haskell,
Dialect::Lisp,
Dialect::Markup,
Dialect::Ml,
Dialect::Percent,
Dialect::VisualBasic,
Dialect::Batch,
Dialect::Fortran,
Dialect::Ini,
Dialect::Unknown,
];
pub fn name(self) -> &'static str {
match self {
Dialect::Unknown => "unknown",
Dialect::CFamily => "c",
Dialect::Hash => "hash",
Dialect::Sql => "sql",
Dialect::Lua => "lua",
Dialect::Haskell => "haskell",
Dialect::Lisp => "lisp",
Dialect::Markup => "markup",
Dialect::Ml => "ml",
Dialect::Percent => "percent",
Dialect::VisualBasic => "vb",
Dialect::Batch => "batch",
Dialect::Fortran => "fortran",
Dialect::Ini => "ini",
}
}
pub fn describe(self) -> &'static str {
match self {
Dialect::Unknown => "no comment syntax detected (input passes through unchanged)",
Dialect::CFamily => "// and /* */ - C, C++, C#, Java, JS, TS, Go, Rust, Swift, PHP",
Dialect::Hash => "# - Python, Ruby, shell, Perl, YAML, TOML, R, Make, Dockerfile",
Dialect::Sql => "-- and /* */ - SQL dialects",
Dialect::Lua => "-- and --[[ ]] - Lua",
Dialect::Haskell => "-- and {- -} - Haskell, Elm, PureScript",
Dialect::Lisp => "; and #| |# - Lisp, Scheme, Clojure, assembly",
Dialect::Markup => "<!-- --> - HTML, XML, SVG, Markdown, Vue, Svelte",
Dialect::Ml => "(* *) - OCaml, F#, Pascal, Mathematica",
Dialect::Percent => "% - TeX, Erlang, Prolog, Matlab/Octave",
Dialect::VisualBasic => "' and REM - Visual Basic, VBA, VBScript",
Dialect::Batch => "REM and :: - Windows batch",
Dialect::Fortran => "! - modern Fortran",
Dialect::Ini => "; and # - INI, desktop entries, gitconfig",
}
}
pub fn from_name(s: &str) -> Option<Dialect> {
let lower = s.to_ascii_lowercase();
Some(match lower.as_str() {
"unknown" | "none" | "plain" => Dialect::Unknown,
"c" | "c-family" | "cfamily" | "clike" | "c-like" => Dialect::CFamily,
"hash" | "pound" | "shell" | "python" => Dialect::Hash,
"sql" => Dialect::Sql,
"lua" => Dialect::Lua,
"haskell" | "hs" | "elm" => Dialect::Haskell,
"lisp" | "scheme" | "clojure" | "asm" => Dialect::Lisp,
"markup" | "html" | "xml" | "md" | "markdown" => Dialect::Markup,
"ml" | "ocaml" | "fsharp" | "pascal" => Dialect::Ml,
"percent" | "tex" | "latex" | "erlang" | "matlab" => Dialect::Percent,
"vb" | "basic" | "vba" | "vbscript" => Dialect::VisualBasic,
"batch" | "bat" | "cmd" => Dialect::Batch,
"fortran" | "f90" => Dialect::Fortran,
"ini" | "conf" | "toml-ini" => Dialect::Ini,
_ => return None,
})
}
pub fn syntax(self) -> Syntax {
Syntax::of(self)
}
}
const C_LINES: &[LineRule] = &[LineRule::new("//").not_after(b"\\:")];
const C_BLOCKS: &[BlockRule] = &[BlockRule::new("/*", "*/")];
const C_STRINGS: &[StringRule] = &[
StringRule::quote("\"\"\"").multiline(), StringRule::quote("\"").raw_prefix(),
StringRule::quote("`").multiline(), StringRule::quote("'").char_like(), ];
const HASH_LINES: &[LineRule] = &[LineRule::new("#").boundary_before()];
const HASH_BLOCKS: &[BlockRule] = &[
BlockRule::new("<#", "#>"), BlockRule::new("#=", "=#").nestable(), BlockRule::new("=begin", "=end").line_anchored(), ];
const HASH_STRINGS: &[StringRule] = &[
StringRule::quote("\"\"\"").multiline().doc_capable(),
StringRule::quote("'''").multiline().doc_capable(),
StringRule::quote("\"").raw_prefix(),
StringRule::quote("'"),
StringRule::quote("`").multiline(), ];
const SQL_LINES: &[LineRule] = &[LineRule::new("--")];
const SQL_BLOCKS: &[BlockRule] = &[BlockRule::new("/*", "*/").nestable()];
const SQL_STRINGS: &[StringRule] = &[
StringRule::quote("'").no_escape().doubling().multiline(),
StringRule::quote("\"").no_escape().doubling(),
StringRule::quote("`").no_escape(), ];
const LUA_LINES: &[LineRule] = &[LineRule::new("--")];
const LUA_STRINGS: &[StringRule] = &[StringRule::quote("\""), StringRule::quote("'")];
const HASKELL_LINES: &[LineRule] = &[LineRule::new("--")];
const HASKELL_BLOCKS: &[BlockRule] = &[BlockRule::new("{-", "-}").nestable()];
const HASKELL_STRINGS: &[StringRule] =
&[StringRule::quote("\""), StringRule::quote("'").char_like()];
const LISP_LINES: &[LineRule] = &[LineRule::new(";").not_after(b"\\")];
const LISP_BLOCKS: &[BlockRule] = &[BlockRule::new("#|", "|#").nestable()];
const LISP_STRINGS: &[StringRule] = &[StringRule::quote("\"").multiline()];
const MARKUP_BLOCKS: &[BlockRule] = &[BlockRule::new("<!--", "-->")];
const MARKUP_STRINGS: &[StringRule] = &[StringRule::quote("`").no_escape()];
const ML_LINES: &[LineRule] = &[LineRule::new("//")]; const ML_BLOCKS: &[BlockRule] = &[BlockRule::new("(*", "*)").nestable()];
const ML_STRINGS: &[StringRule] = &[StringRule::quote("\""), StringRule::quote("'").char_like()];
const PERCENT_LINES: &[LineRule] = &[LineRule::new("%").not_after(b"\\")];
const PERCENT_BLOCKS: &[BlockRule] = &[BlockRule::new("%{", "%}").line_anchored()];
const PERCENT_STRINGS: &[StringRule] = &[StringRule::quote("\"")];
const VB_LINES: &[LineRule] = &[LineRule::new("'"), LineRule::new("REM").word()];
const VB_STRINGS: &[StringRule] = &[StringRule::quote("\"").no_escape().doubling()];
const BATCH_LINES: &[LineRule] = &[
LineRule::new("REM").word().anchored(),
LineRule::new("::").anchored(),
];
const BATCH_STRINGS: &[StringRule] = &[StringRule::quote("\"").no_escape()];
const FORTRAN_LINES: &[LineRule] = &[LineRule::new("!")];
const FORTRAN_STRINGS: &[StringRule] = &[
StringRule::quote("'").no_escape().doubling(),
StringRule::quote("\"").no_escape().doubling(),
];
const INI_LINES: &[LineRule] = &[
LineRule::new(";").boundary_before(),
LineRule::new("#").boundary_before(),
];
const INI_STRINGS: &[StringRule] = &[StringRule::quote("\""), StringRule::quote("'")];
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Syntax {
pub dialect: Dialect,
pub lines: &'static [LineRule],
pub blocks: &'static [BlockRule],
pub strings: &'static [StringRule],
pub long_brackets: bool,
pub html_embedded: bool,
pub shebang_is_code: bool,
pub char_prefix: Option<&'static str>,
}
impl Syntax {
pub const fn of(dialect: Dialect) -> Syntax {
let base = Syntax {
dialect,
lines: &[],
blocks: &[],
strings: &[],
long_brackets: false,
html_embedded: false,
shebang_is_code: true,
char_prefix: None,
};
match dialect {
Dialect::Unknown => base,
Dialect::CFamily => Syntax {
lines: C_LINES,
blocks: C_BLOCKS,
strings: C_STRINGS,
..base
},
Dialect::Hash => Syntax {
lines: HASH_LINES,
blocks: HASH_BLOCKS,
strings: HASH_STRINGS,
..base
},
Dialect::Sql => Syntax {
lines: SQL_LINES,
blocks: SQL_BLOCKS,
strings: SQL_STRINGS,
..base
},
Dialect::Lua => Syntax {
lines: LUA_LINES,
strings: LUA_STRINGS,
long_brackets: true,
..base
},
Dialect::Haskell => Syntax {
lines: HASKELL_LINES,
blocks: HASKELL_BLOCKS,
strings: HASKELL_STRINGS,
..base
},
Dialect::Lisp => Syntax {
lines: LISP_LINES,
blocks: LISP_BLOCKS,
strings: LISP_STRINGS,
char_prefix: Some("#\\"),
..base
},
Dialect::Markup => Syntax {
blocks: MARKUP_BLOCKS,
strings: MARKUP_STRINGS,
html_embedded: true,
..base
},
Dialect::Ml => Syntax {
lines: ML_LINES,
blocks: ML_BLOCKS,
strings: ML_STRINGS,
..base
},
Dialect::Percent => Syntax {
lines: PERCENT_LINES,
blocks: PERCENT_BLOCKS,
strings: PERCENT_STRINGS,
..base
},
Dialect::VisualBasic => Syntax {
lines: VB_LINES,
strings: VB_STRINGS,
..base
},
Dialect::Batch => Syntax {
lines: BATCH_LINES,
strings: BATCH_STRINGS,
..base
},
Dialect::Fortran => Syntax {
lines: FORTRAN_LINES,
strings: FORTRAN_STRINGS,
..base
},
Dialect::Ini => Syntax {
lines: INI_LINES,
strings: INI_STRINGS,
..base
},
}
}
pub fn is_empty(&self) -> bool {
self.lines.is_empty() && self.blocks.is_empty() && !self.long_brackets
}
pub(crate) fn interesting(&self) -> [bool; 256] {
let mut t = [false; 256];
let mut mark = |s: &str, ci: bool| {
if let Some(&b) = s.as_bytes().first() {
t[b as usize] = true;
if ci {
t[b.to_ascii_lowercase() as usize] = true;
t[b.to_ascii_uppercase() as usize] = true;
}
}
};
for r in self.lines {
mark(r.marker, r.ci);
}
for r in self.blocks {
mark(r.open, false);
}
for r in self.strings {
mark(r.open, false);
}
if let Some(p) = self.char_prefix {
mark(p, false);
}
if self.long_brackets {
t[b'[' as usize] = true;
}
if self.html_embedded {
t[b'<' as usize] = true;
}
t
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_dialect_round_trips_through_its_name() {
for &d in Dialect::ALL {
assert_eq!(Dialect::from_name(d.name()), Some(d), "{}", d.name());
}
}
#[test]
fn longer_markers_are_ordered_before_their_prefixes() {
for &d in Dialect::ALL {
let syn = d.syntax();
for (i, a) in syn.strings.iter().enumerate() {
for b in &syn.strings[i + 1..] {
assert!(
!b.open.starts_with(a.open) || a.open == b.open,
"{}: {:?} shadows {:?}",
d.name(),
a.open,
b.open
);
}
}
}
}
#[test]
fn unknown_syntax_recognises_nothing() {
assert!(Dialect::Unknown.syntax().is_empty());
assert!(!Dialect::CFamily.syntax().is_empty());
}
}