use crate::syntax::{Dialect, Syntax};
pub const THRESHOLD: i32 = 16;
const SAMPLE_LIMIT: usize = 256 * 1024;
#[derive(Clone, Debug)]
pub struct Detection {
pub dialect: Dialect,
pub confidence: f32,
pub scores: Vec<(Dialect, i32)>,
}
impl Detection {
pub fn syntax(&self) -> Syntax {
self.dialect.syntax()
}
}
pub fn detect(src: &str) -> Detection {
let ev = Evidence::collect(src);
let mut scores: Vec<(Dialect, i32)> = Dialect::ALL
.iter()
.filter(|d| **d != Dialect::Unknown)
.map(|&d| (d, score(d, &ev)))
.collect();
scores.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.name().cmp(b.0.name())));
if ev.kw_markup >= 2
|| (ev.kw_markup >= 1 && ev.ls_lt >= 2)
|| ev.md_fence >= 2
|| (ev.md_fence >= 1 && ev.kw_md >= 3)
{
return Detection {
dialect: Dialect::Markup,
confidence: 1.0,
scores,
};
}
let (best_d, best) = scores[0];
let second = scores.get(1).map(|s| s.1).unwrap_or(0).max(0);
if best < THRESHOLD {
return Detection {
dialect: Dialect::Unknown,
confidence: 0.0,
scores,
};
}
let strength = (best as f32 / 60.0).min(1.0);
let margin = 1.0 - (second as f32 / best as f32);
Detection {
dialect: best_d,
confidence: (strength * margin).clamp(0.0, 1.0),
scores,
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
enum Shebang {
#[default]
None,
HashLang,
CLang,
}
#[derive(Debug, Default)]
struct Evidence {
lines: u32,
shebang: Shebang,
ls_slash2: u32,
ls_star: u32,
ls_hash: u32,
ls_dash2: u32,
ls_semi: u32,
ls_percent: u32,
ls_squote: u32,
ls_bang: u32,
ls_ltbang: u32,
ls_parenstar: u32,
ls_bracedash: u32,
ls_hashpipe: u32,
ls_rem: u32,
ls_colon2: u32,
ls_lt: u32,
ls_paren: u32,
slashstar: u32,
starslash: u32,
trailing_slash2: u32,
trailing_dash2: u32,
trailing_hash: u32,
trailing_semi: u32,
trailing_squote: u32,
trailing_bang: u32,
parenstar: u32,
starparen: u32,
bracedash: u32,
dashbrace: u32,
hashpipe: u32,
pipehash: u32,
ltbang: u32,
end_semi: u32,
end_brace: u32,
end_colon: u32,
kv_lines: u32,
preproc: u32,
kw_c: u32,
kw_hash: u32,
kw_sql: u32,
kw_lua: u32,
kw_hs: u32,
kw_lisp: u32,
kw_markup: u32,
kw_md: u32,
kw_ml: u32,
kw_tex: u32,
kw_erlang: u32,
kw_matlab: u32,
kw_vb: u32,
kw_batch: u32,
kw_fortran: u32,
md_fence: u32,
lua_long: u32,
ruby_block: u32,
ps_block: u32,
ini_section: u32,
}
impl Evidence {
fn collect(src: &str) -> Evidence {
let mut e = Evidence::default();
let sample = &src[..floor_char_boundary(src, SAMPLE_LIMIT)];
let mut in_triple: Option<&[u8]> = None;
for (n, raw) in sample.split('\n').enumerate() {
let line = raw.strip_suffix('\r').unwrap_or(raw);
e.lines += 1;
let t = line.trim_start().as_bytes();
if t.is_empty() {
continue;
}
if let Some(delim) = in_triple {
if odd_count(line.as_bytes(), delim) {
in_triple = None;
}
continue;
}
if n == 0 && t.starts_with(b"#!") {
e.shebang = shebang_kind(line);
continue;
}
e.leading(t);
e.trailing(t);
e.inline(t);
for delim in [b"\"\"\"".as_slice(), b"'''".as_slice()] {
if odd_count(line.as_bytes(), delim) {
in_triple = Some(delim);
break;
}
}
}
e
}
fn leading(&mut self, t: &[u8]) {
match t[0] {
b'/' => {
if t.starts_with(b"//") {
self.ls_slash2 += 1;
} else if t.starts_with(b"/*") {
self.kw_c += 1;
}
}
b'*' => self.ls_star += 1,
b'#' => {
if is_preproc(t) {
self.preproc += 1;
} else if t.starts_with(b"#|") {
self.ls_hashpipe += 1;
} else if !t.ends_with(b"\\") {
self.ls_hash += 1;
if t.starts_with(b"##") {
self.kw_md += 1;
}
}
}
b'-' => {
if t.starts_with(b"--[") {
self.lua_long += 1;
self.ls_dash2 += 1;
} else if is_dash_comment(t) {
self.ls_dash2 += 1;
} else if t.starts_with(b"-module(") || t.starts_with(b"-export(") {
self.kw_erlang += 1;
}
}
b';' => {
let run = t.iter().take_while(|c| **c == b';').count();
let rest = t[run..].trim_ascii();
if run > 2 || (!rest.is_empty() && rest != b"\\") {
self.ls_semi += 1;
}
}
b'%' => {
if t.get(1).is_none_or(|c| matches!(c, b' ' | b'%'))
&& !t[1..].trim_ascii().is_empty()
{
self.ls_percent += 1;
}
if t == b"%{" || t == b"%}" {
self.kw_matlab += 1;
}
}
b'\'' => self.ls_squote += 1,
b'!' => self.ls_bang += 1,
b'<' => {
if t.starts_with(b"<!--") {
self.ls_ltbang += 1;
} else if starts_ci(t, "<?xml")
|| starts_ci(t, "<!doctype")
|| starts_ci(t, "<html")
{
self.kw_markup += 1;
} else if t.starts_with(b"<#") {
self.ps_block += 1;
} else {
self.ls_lt += 1;
}
}
b'(' => {
self.ls_paren += 1;
if t.starts_with(b"(*") {
if marker_followed_by_text(t, 2, b"*") {
self.ls_parenstar += 1;
}
} else if t.starts_with(b"(def")
|| t.starts_with(b"(ns ")
|| t.starts_with(b"(let")
|| t.starts_with(b"(require")
|| t.starts_with(b"(fn ")
{
self.kw_lisp += 1;
}
}
b'{' => {
if t.starts_with(b"{-") {
self.ls_bracedash += 1;
}
}
b'=' => {
if t.starts_with(b"=begin") || t.starts_with(b"=end") {
self.ruby_block += 1;
}
}
b'[' => {
if t.ends_with(b"]") && t.len() > 2 && !t.contains(&b',') {
self.ini_section += 1;
}
}
b'`' => {
if t.starts_with(b"```") {
self.kw_md += 1;
self.md_fence += 1;
}
}
b'\\' => {
if t.get(1).is_some_and(|c| c.is_ascii_alphabetic()) && t.contains(&b'{') {
self.kw_tex += 1;
}
}
b'@' => {
if starts_ci(t, "@echo") {
self.kw_batch += 1;
}
}
b':' => {
if t.starts_with(b"::") && marker_followed_by_text(t, 2, b":") {
self.ls_colon2 += 1;
}
}
_ => self.keywords(t),
}
}
fn keywords(&mut self, t: &[u8]) {
if starts_ci(t, "rem") && t.get(3).is_none_or(|c| *c == b' ' || *c == b'\t') {
self.ls_rem += 1;
}
for (pat, field) in [
("public ", 0u8),
("private ", 0),
("protected ", 0),
("void ", 0),
("int ", 0),
("char ", 0),
("struct ", 0),
("typedef ", 0),
("namespace ", 0),
("func ", 0),
("fn ", 0),
("impl ", 0),
("var ", 0),
("let ", 0),
("const ", 0),
("function(", 0),
("function ", 0),
("console.", 0),
("System.out", 0),
("std::", 0),
("def ", 1),
("elif ", 1),
("print(", 1),
("echo ", 1),
("puts ", 1),
("fi", 1),
("esac", 1),
("done", 1),
("local ", 3),
("elseif ", 3),
("then", 3),
("nil", 3),
("module ", 4),
("newtype ", 4),
("instance ", 4),
("data ", 4),
("where", 4),
("let rec", 5),
("match ", 5),
("type ", 5),
("begin", 5),
("implicit none", 6),
("subroutine ", 6),
("end program", 6),
("integer ::", 6),
("real ::", 6),
] {
if starts_word(t, pat) {
match field {
0 => self.kw_c += 1,
1 => self.kw_hash += 1,
3 => self.kw_lua += 1,
4 => self.kw_hs += 1,
5 => self.kw_ml += 1,
_ => self.kw_fortran += 1,
}
break;
}
}
for pat in [
"select ",
"insert into",
"update ",
"create table",
"alter table",
"delete from",
] {
if starts_ci(t, pat) {
self.kw_sql += 1;
break;
}
}
for pat in [
"dim ",
"end sub",
"end function",
"option explicit",
"private sub",
"public sub",
] {
if starts_ci(t, pat) {
self.kw_vb += 1;
break;
}
}
for pat in ["setlocal", "if exist", "@echo", "goto :eof"] {
if starts_ci(t, pat) {
self.kw_batch += 1;
break;
}
}
if t.starts_with(b"program ") || t.starts_with(b"end module") {
self.kw_fortran += 1;
}
if let Some(sep) = t.iter().position(|c| *c == b'=' || *c == b':') {
let key = t[..sep].trim_ascii_end();
let plain = !key.is_empty()
&& key
.iter()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, b'_' | b'-' | b'.'));
if plain {
self.kv_lines += 1;
}
}
}
fn trailing(&mut self, t: &[u8]) {
let end = t.iter().rposition(|c| *c != b' ' && *c != b'\t');
if let Some(i) = end {
match t[i] {
b';' => self.end_semi += 1,
b'{' | b'}' => self.end_brace += 1,
b':' => self.end_colon += 1,
_ => {}
}
}
}
fn inline(&mut self, t: &[u8]) {
let mut i = 0;
while i < t.len() {
let (a, b) = (t[i], t.get(i + 1).copied().unwrap_or(0));
let prev_ws = i > 0 && (t[i - 1] == b' ' || t[i - 1] == b'\t');
match (a, b) {
(b'/', b'*') => {
if marker_followed_by_text(t, i + 2, b"*") {
self.slashstar += 1;
}
}
(b'*', b'/') => self.starslash += 1,
(b'/', b'/') => {
if prev_ws {
self.trailing_slash2 += 1;
}
i += 1;
}
(b'-', b'-') => {
if prev_ws && is_dash_comment(&t[i..]) {
self.trailing_dash2 += 1;
}
i += 1;
}
(b'(', b'*') => {
if t.get(i + 2)
.is_none_or(|c| matches!(c, b' ' | b'\t' | b'*'))
{
self.parenstar += 1;
}
}
(b'*', b')') => self.starparen += 1,
(b'{', b'-') => self.bracedash += 1,
(b'-', b'}') => self.dashbrace += 1,
(b'#', b'|') => self.hashpipe += 1,
(b'|', b'#') => self.pipehash += 1,
(b'<', b'!') => {
if t[i..].starts_with(b"<!--") {
self.ltbang += 1;
}
}
(b'#', _) => {
if prev_ws && !is_preproc(&t[i..]) {
self.trailing_hash += 1;
}
}
(b';', _) if prev_ws => self.trailing_semi += 1,
(b'\'', _) if prev_ws => self.trailing_squote += 1,
(b'!', b'=') => {}
(b'!', _) if prev_ws => self.trailing_bang += 1,
(b'[', b'(') => self.kw_md += 1, _ => {}
}
i += 1;
}
}
}
fn is_dash_comment(t: &[u8]) -> bool {
t.starts_with(b"--")
&& t.get(2)
.is_none_or(|c| matches!(c, b' ' | b'\t' | b'[' | b'|' | b'\r'))
}
fn odd_count(line: &[u8], delim: &[u8]) -> bool {
let mut n = 0usize;
let mut i = 0;
while i + delim.len() <= line.len() {
if &line[i..i + delim.len()] == delim {
n += 1;
i += delim.len();
} else {
i += 1;
}
}
n % 2 == 1
}
fn marker_followed_by_text(t: &[u8], at: usize, repeats: &[u8]) -> bool {
t.get(at)
.is_none_or(|c| matches!(c, b' ' | b'\t' | b'\r') || repeats.contains(c))
}
fn is_preproc(t: &[u8]) -> bool {
const DIRECTIVES: [&str; 21] = [
"include_next",
"include",
"import",
"define",
"undef",
"ifdef",
"ifndef",
"if",
"elifdef",
"elifndef",
"elif",
"else",
"endif",
"error",
"warning",
"pragma",
"line",
"assert",
"unassert",
"ident",
"sccs",
];
if !t.starts_with(b"#") {
return false;
}
let rest = &t[1..];
let start = rest
.iter()
.position(|c| *c != b' ' && *c != b'\t')
.unwrap_or(rest.len());
let word = &rest[start..];
DIRECTIVES.iter().any(|d| {
word.starts_with(d.as_bytes())
&& word
.get(d.len())
.is_none_or(|c| !c.is_ascii_alphanumeric() && *c != b'_')
})
}
fn starts_word(t: &[u8], pat: &str) -> bool {
let p = pat.as_bytes();
if !t.starts_with(p) {
return false;
}
match p.last() {
Some(c) if c.is_ascii_alphanumeric() || *c == b'_' => t
.get(p.len())
.is_none_or(|c| !c.is_ascii_alphanumeric() && *c != b'_'),
_ => true,
}
}
fn starts_ci(t: &[u8], pat: &str) -> bool {
t.len() >= pat.len() && t[..pat.len()].eq_ignore_ascii_case(pat.as_bytes())
}
fn shebang_kind(line: &str) -> Shebang {
let l = line.to_ascii_lowercase();
const C_LIKE: [&str; 3] = ["node", "deno", "bun"];
if C_LIKE.iter().any(|n| l.contains(n)) {
return Shebang::CLang;
}
Shebang::HashLang
}
fn floor_char_boundary(s: &str, mut i: usize) -> usize {
if i >= s.len() {
return s.len();
}
while !s.is_char_boundary(i) {
i -= 1;
}
i
}
fn cap(v: u32, max: u32) -> i32 {
v.min(max) as i32
}
struct Case {
markers: i32,
bonus: i32,
penalty: i32,
}
fn score(d: Dialect, e: &Evidence) -> i32 {
match d {
Dialect::VisualBasic if e.kw_vb < 2 => return 0,
Dialect::Fortran if e.kw_fortran < 2 || e.ls_bang == 0 => return 0,
Dialect::Ini if e.ini_section == 0 => return 0,
Dialect::Percent if e.kw_tex + e.kw_erlang + e.kw_matlab == 0 => return 0,
Dialect::Ml if e.ls_parenstar == 0 => return 0,
Dialect::Sql if e.ls_dash2 == 0 && e.slashstar.min(e.starslash) == 0 => return 0,
Dialect::Lua if e.ls_dash2 == 0 && e.lua_long == 0 => return 0,
Dialect::Haskell if e.ls_dash2 == 0 && e.bracedash.min(e.dashbrace) == 0 => return 0,
Dialect::Markup if e.ls_ltbang == 0 && e.kw_markup == 0 && e.ls_lt < 3 => return 0,
Dialect::Batch if e.kw_batch == 0 && e.ls_rem == 0 => return 0,
_ => {}
}
let c = case(d, e);
if c.markers == 0 {
return 0;
}
c.markers + c.bonus.min(c.markers) - c.penalty
}
fn case(d: Dialect, e: &Evidence) -> Case {
let code_shape = cap(e.end_semi + e.end_brace, 40);
match d {
Dialect::Unknown => Case {
markers: 0,
bonus: 0,
penalty: 0,
},
Dialect::CFamily => Case {
markers: 14 * cap(e.ls_slash2, 20)
+ 12 * cap(e.slashstar.min(e.starslash), 20)
+ 5 * cap(e.trailing_slash2, 10),
bonus: 4 * cap(e.ls_star, 10)
+ 2 * code_shape
+ 20 * cap(e.preproc, 3)
+ 3 * cap(e.kw_c, 15)
+ if e.shebang == Shebang::CLang { 30 } else { 0 },
penalty: 5 * cap(e.kw_hash, 10)
+ 2 * cap(e.end_colon, 20)
+ if e.ls_hash > 0 && e.preproc == 0 && e.ls_slash2 == 0 && e.slashstar == 0 {
40
} else {
0
},
},
Dialect::Hash => Case {
markers: 14 * cap(e.ls_hash, 20)
+ 5 * cap(e.trailing_hash, 10)
+ 12 * cap(e.ruby_block, 2)
+ 12 * cap(e.ps_block, 2),
bonus: 3 * cap(e.kw_hash, 15)
+ 2 * cap(e.end_colon, 20)
+ 2 * cap(e.kv_lines, 30)
+ if e.shebang == Shebang::HashLang {
30
} else {
0
},
penalty: 3 * code_shape
+ 25 * cap(e.preproc, 3)
+ 20 * cap(e.kw_md, 10)
+ 6 * cap(e.ls_slash2, 10),
},
Dialect::Sql => Case {
markers: 14 * cap(e.ls_dash2, 20)
+ 5 * cap(e.trailing_dash2, 10)
+ 12 * cap(e.slashstar.min(e.starslash), 10),
bonus: 10 * cap(e.kw_sql, 10),
penalty: 30 * cap(e.lua_long, 1) + 8 * cap(e.kw_hs, 5) + 6 * cap(e.kw_lua, 5),
},
Dialect::Lua => Case {
markers: 14 * cap(e.ls_dash2, 20)
+ 5 * cap(e.trailing_dash2, 10)
+ 25 * cap(e.lua_long, 3),
bonus: 6 * cap(e.kw_lua, 10),
penalty: 8 * cap(e.kw_sql, 5) + 8 * cap(e.kw_hs, 5),
},
Dialect::Haskell => Case {
markers: 14 * cap(e.ls_dash2, 20)
+ 5 * cap(e.trailing_dash2, 10)
+ 20 * cap(e.bracedash.min(e.dashbrace), 5),
bonus: 6 * cap(e.kw_hs, 10),
penalty: 8 * cap(e.kw_sql, 5) + 8 * cap(e.kw_lua, 5),
},
Dialect::Lisp => Case {
markers: 14 * cap(e.ls_semi, 20) + 15 * cap(e.hashpipe.min(e.pipehash), 5),
bonus: 5 * cap(e.kw_lisp, 15) + 2 * cap(e.ls_paren, 20),
penalty: 20 * cap(e.ini_section, 1) + 2 * code_shape,
},
Dialect::Markup => Case {
markers: 16 * cap(e.ls_ltbang + e.ltbang, 20),
bonus: 25 * cap(e.kw_markup, 3) + 6 * cap(e.kw_md, 10) + cap(e.ls_lt, 30),
penalty: 2 * cap(e.kv_lines, 20) + 4 * cap(e.kw_hash, 15) + 2 * cap(e.end_colon, 20),
},
Dialect::Ml => Case {
markers: 14 * cap(e.ls_parenstar, 20) + 12 * cap(e.parenstar.min(e.starparen), 10),
bonus: 5 * cap(e.kw_ml, 10),
penalty: if e.preproc > 0 || e.ls_slash2 > 0 || e.end_semi > 3 {
40
} else {
0
},
},
Dialect::Percent => Case {
markers: 14 * cap(e.ls_percent, 20),
bonus: 30 * cap(e.kw_tex, 3) + 30 * cap(e.kw_erlang, 3) + 15 * cap(e.kw_matlab, 3),
penalty: if e.kw_tex == 0 && e.kw_erlang == 0 && e.end_semi + e.end_brace > 5 {
25
} else {
0
},
},
Dialect::VisualBasic => Case {
markers: 10 * cap(e.ls_squote, 20)
+ 5 * cap(e.trailing_squote, 10)
+ 12 * cap(e.ls_rem, 10),
bonus: 25 * cap(e.kw_vb, 4),
penalty: 0,
},
Dialect::Batch => Case {
markers: 12 * cap(e.ls_rem, 10) + 12 * cap(e.ls_colon2, 10),
bonus: 20 * cap(e.kw_batch, 3),
penalty: 0,
},
Dialect::Fortran => Case {
markers: 10 * cap(e.ls_bang, 20) + 5 * cap(e.trailing_bang, 10),
bonus: 25 * cap(e.kw_fortran, 4),
penalty: 0,
},
Dialect::Ini => Case {
markers: 10 * cap(e.ls_semi, 10)
+ 10 * cap(e.ls_hash, 10)
+ 5 * cap(e.trailing_semi + e.trailing_hash, 10),
bonus: 12 * cap(e.ini_section, 10) + cap(e.kv_lines, 20),
penalty: 3 * code_shape + 2 * cap(e.ls_paren, 20),
},
}
}
#[cfg(test)]
mod tests {
use super::*;
fn d(src: &str) -> Dialect {
detect(src).dialect
}
#[test]
fn c_family_from_slashes_and_braces() {
assert_eq!(
d("int main(void) {\n // start\n return 0;\n}\n"),
Dialect::CFamily
);
assert_eq!(
d("#include <stdio.h>\n#define X 1\n/* note */\nint x = 3 % 2;\n"),
Dialect::CFamily
);
}
#[test]
fn structure_alone_never_licenses_stripping() {
assert_eq!(
d("#include <stdio.h>\n#define X 1\nint x = 3 % 2;\n"),
Dialect::Unknown
);
assert_eq!(d("#!/bin/bash\nset -e\necho hi\n"), Dialect::Unknown);
}
#[test]
fn python_floor_division_is_not_a_comment() {
let src = "def f(a, b):\n return a // b\n";
assert_eq!(d(src), Dialect::Unknown);
assert_eq!(crate::strip_comments(src), src);
let src = "# halve it\ndef f(a, b):\n return a // b\n";
assert_eq!(d(src), Dialect::Hash);
assert_eq!(
crate::strip_comments(src),
"def f(a, b):\n return a // b\n"
);
}
#[test]
fn shell_and_python_are_hash() {
assert_eq!(
d("#!/bin/bash\n# strict mode\nset -e\necho hi\n"),
Dialect::Hash
);
assert_eq!(d("# config\nname: thing\nport: 80\n"), Dialect::Hash);
}
#[test]
fn markdown_headings_are_not_shell_comments() {
let src =
"# Title\n\nSome text.\n\n## Section\n\n- a bullet\n- another\n\n```sh\nls\n```\n";
assert_ne!(d(src), Dialect::Hash);
}
#[test]
fn markup_wins_for_html() {
assert_eq!(
d("<!DOCTYPE html>\n<html>\n<!-- hi -->\n<body></body>\n</html>\n"),
Dialect::Markup
);
}
#[test]
fn sql_lua_and_haskell_share_a_marker_but_split_on_context() {
assert_eq!(
d("-- fetch rows\nSELECT a, b FROM t WHERE x = 1;\n"),
Dialect::Sql
);
assert_eq!(
d("-- init\nlocal x = 1\nfunction f()\n return x\nend\n"),
Dialect::Lua
);
assert_eq!(
d("-- | doc\nmodule Main where\n{- block -}\ndata T = T\n"),
Dialect::Haskell
);
}
#[test]
fn empty_and_markerless_input_is_unknown() {
assert_eq!(d(""), Dialect::Unknown);
assert_eq!(d("just some words\nand more words\n"), Dialect::Unknown);
}
#[test]
fn confidence_is_bounded() {
for src in ["", "// x\n", "#!/bin/sh\n# y\n", "<!-- z -->"] {
let det = detect(src);
assert!((0.0..=1.0).contains(&det.confidence), "{src:?} -> {det:?}");
}
}
}