diffler_core/
language.rs

1//! A path's language, its Linguist colour, and its comment syntax for the
2//! line counter. The syntax registry owns the extension table for every
3//! language it highlights; this module adds the ones it only counts.
4
5use crate::syntax::registry::REGISTRY;
6
7pub type Rgb = (u8, u8, u8);
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub struct Language {
11    pub name: &'static str,
12    /// Linguist's colour, unadjusted.
13    pub color: Rgb,
14    line_comments: &'static [&'static str],
15    block_comment: Option<(&'static str, &'static str)>,
16}
17
18impl Language {
19    /// The kind of a trimmed `line`, and whether a block comment is still
20    /// open after it.
21    fn classify(self, line: &str, in_block: bool) -> (LineKind, bool) {
22        if in_block {
23            let closed = self
24                .block_comment
25                .is_some_and(|(_, end)| line.contains(end));
26            return (LineKind::Comment, !closed);
27        }
28        // we test the block opener first since Lua's `--[[` starts with `--`
29        if let Some((start, end)) = self.block_comment
30            && line.starts_with(start)
31        {
32            let closed = line[start.len()..].contains(end);
33            return (LineKind::Comment, !closed);
34        }
35        if self
36            .line_comments
37            .iter()
38            .any(|token| line.starts_with(token))
39        {
40            return (LineKind::Comment, false);
41        }
42        (LineKind::Code, false)
43    }
44}
45
46enum LineKind {
47    Code,
48    Comment,
49}
50
51/// `(code, comments, blanks)`. A line counts as a comment only when it opens
52/// with one, as `scc` and `cloc` count.
53#[must_use]
54pub fn count_lines(text: &str, language: Option<Language>) -> (usize, usize, usize) {
55    let (mut code, mut comments, mut blanks) = (0, 0, 0);
56    let mut in_block = false;
57    for (index, raw) in text.lines().enumerate() {
58        let line = raw.trim();
59        if line.is_empty() {
60            blanks += 1;
61            continue;
62        }
63        // a shebang is code, though `#` opens a comment in the same languages
64        if index == 0 && line.starts_with("#!") {
65            code += 1;
66            continue;
67        }
68        let Some(language) = language else {
69            code += 1;
70            continue;
71        };
72        let (kind, still_open) = language.classify(line, in_block);
73        in_block = still_open;
74        match kind {
75            LineKind::Code => code += 1,
76            LineKind::Comment => comments += 1,
77        }
78    }
79    (code, comments, blanks)
80}
81
82#[must_use]
83pub fn of_path(path: &str) -> Option<Language> {
84    if let Some(entry) = REGISTRY.for_path(path)
85        && let Some(language) = by_key(entry.name)
86    {
87        return Some(language);
88    }
89    let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase();
90    if let Some(key) = EXTRA_FILENAMES
91        .iter()
92        .find_map(|(filename, key)| (*filename == name).then_some(key))
93    {
94        return by_key(key);
95    }
96    if let Some(extension) = name.rsplit_once('.').map(|(_, ext)| ext)
97        && let Some(key) = EXTRA_EXTENSIONS
98            .iter()
99            .find_map(|(ext, key)| (*ext == extension).then_some(key))
100    {
101        return by_key(key);
102    }
103    // we check doc names after extensions so `LICENSE.md` stays Markdown
104    if crate::classify::DOC_NAMES
105        .iter()
106        .any(|stem| name.starts_with(stem))
107    {
108        return by_key("text");
109    }
110    None
111}
112
113fn by_key(key: &str) -> Option<Language> {
114    TABLE
115        .iter()
116        .find_map(|(name, language)| (*name == key).then_some(*language))
117}
118
119/// Push `color` away from `bg` until it reaches 3:1 contrast. Linguist tunes
120/// its palette for a white page, so JSON's `#292929` vanishes on a dark one.
121#[must_use]
122pub fn readable_on(color: Rgb, bg: Rgb) -> Rgb {
123    const TARGET: f32 = 3.0;
124    let toward = if luminance(bg) > 0.5 { 0 } else { 255 };
125    let mut out = color;
126    for _ in 0..40 {
127        if contrast(out, bg) >= TARGET {
128            break;
129        }
130        out = step_toward(out, toward);
131    }
132    out
133}
134
135fn step_toward(color: Rgb, toward: u8) -> Rgb {
136    let blend = |channel: u8| {
137        let (from, to) = (i32::from(channel), i32::from(toward));
138        let by = (to - from) / 10;
139        let stepped = from + if by == 0 { (to - from).signum() } else { by };
140        u8::try_from(stepped.clamp(0, 255)).unwrap_or(toward)
141    };
142    (blend(color.0), blend(color.1), blend(color.2))
143}
144
145fn contrast(a: Rgb, b: Rgb) -> f32 {
146    let (high, low) = {
147        let (la, lb) = (luminance(a), luminance(b));
148        if la > lb { (la, lb) } else { (lb, la) }
149    };
150    (high + 0.05) / (low + 0.05)
151}
152
153/// WCAG relative luminance.
154fn luminance(color: Rgb) -> f32 {
155    let channel = |value: u8| {
156        let value = f32::from(value) / 255.0;
157        if value <= 0.03928 {
158            value / 12.92
159        } else {
160            ((value + 0.055) / 1.055).powf(2.4)
161        }
162    };
163    0.2126 * channel(color.0) + 0.7152 * channel(color.1) + 0.0722 * channel(color.2)
164}
165
166/// Keys match `syntax::registry` names where a grammar exists.
167const TABLE: &[(&str, Language)] = &[
168    lang(
169        "rust",
170        "Rust",
171        (0xde, 0xa5, 0x84),
172        &["//"],
173        Some(("/*", "*/")),
174    ),
175    lang(
176        "python",
177        "Python",
178        (0x35, 0x72, 0xa5),
179        &["#"],
180        Some(("\"\"\"", "\"\"\"")),
181    ),
182    lang(
183        "javascript",
184        "JavaScript",
185        (0xf1, 0xe0, 0x5a),
186        &["//"],
187        Some(("/*", "*/")),
188    ),
189    lang(
190        "typescript",
191        "TypeScript",
192        (0x31, 0x78, 0xc6),
193        &["//"],
194        Some(("/*", "*/")),
195    ),
196    lang(
197        "tsx",
198        "TSX",
199        (0x31, 0x78, 0xc6),
200        &["//"],
201        Some(("/*", "*/")),
202    ),
203    lang("go", "Go", (0x00, 0xad, 0xd8), &["//"], Some(("/*", "*/"))),
204    lang("c", "C", (0x55, 0x55, 0x55), &["//"], Some(("/*", "*/"))),
205    lang(
206        "cpp",
207        "C++",
208        (0xf3, 0x4b, 0x7d),
209        &["//"],
210        Some(("/*", "*/")),
211    ),
212    lang(
213        "java",
214        "Java",
215        (0xb0, 0x72, 0x19),
216        &["//"],
217        Some(("/*", "*/")),
218    ),
219    lang(
220        "c-sharp",
221        "C#",
222        (0x17, 0x86, 0x00),
223        &["//"],
224        Some(("/*", "*/")),
225    ),
226    lang(
227        "ruby",
228        "Ruby",
229        (0x70, 0x15, 0x16),
230        &["#"],
231        Some(("=begin", "=end")),
232    ),
233    lang(
234        "php",
235        "PHP",
236        (0x4f, 0x5d, 0x95),
237        &["//", "#"],
238        Some(("/*", "*/")),
239    ),
240    lang("bash", "Shell", (0x89, 0xe0, 0x51), &["#"], None),
241    lang("json", "JSON", (0x29, 0x29, 0x29), &[], None),
242    lang(
243        "html",
244        "HTML",
245        (0xe3, 0x4c, 0x26),
246        &[],
247        Some(("<!--", "-->")),
248    ),
249    lang("css", "CSS", (0x66, 0x33, 0x99), &[], Some(("/*", "*/"))),
250    lang("yaml", "YAML", (0xcb, 0x17, 0x1e), &["#"], None),
251    lang(
252        "sql",
253        "SQL",
254        (0xe3, 0x8c, 0x00),
255        &["--"],
256        Some(("/*", "*/")),
257    ),
258    lang(
259        "markdown",
260        "Markdown",
261        (0x08, 0x3f, 0xa1),
262        &[],
263        Some(("<!--", "-->")),
264    ),
265    lang("toml", "TOML", (0x9c, 0x42, 0x21), &["#"], None),
266    lang(
267        "hcl",
268        "HCL",
269        (0x84, 0x4f, 0xba),
270        &["#", "//"],
271        Some(("/*", "*/")),
272    ),
273    lang("dockerfile", "Dockerfile", (0x38, 0x4d, 0x54), &["#"], None),
274    lang("make", "Makefile", (0x42, 0x78, 0x19), &["#"], None),
275    lang(
276        "lua",
277        "Lua",
278        (0x00, 0x00, 0x80),
279        &["--"],
280        Some(("--[[", "]]")),
281    ),
282    lang("nix", "Nix", (0x7e, 0x7e, 0xff), &["#"], Some(("/*", "*/"))),
283    lang("xml", "XML", (0x00, 0x60, 0xac), &[], Some(("<!--", "-->"))),
284    lang(
285        "swift",
286        "Swift",
287        (0xf0, 0x51, 0x38),
288        &["//"],
289        Some(("/*", "*/")),
290    ),
291    lang(
292        "scala",
293        "Scala",
294        (0xc2, 0x2d, 0x40),
295        &["//"],
296        Some(("/*", "*/")),
297    ),
298    lang(
299        "elixir",
300        "Elixir",
301        (0x6e, 0x4a, 0x7e),
302        &["#"],
303        Some(("@doc \"\"\"", "\"\"\"")),
304    ),
305    lang("zig", "Zig", (0xec, 0x91, 0x5c), &["//"], None),
306    lang(
307        "haskell",
308        "Haskell",
309        (0x5e, 0x50, 0x86),
310        &["--"],
311        Some(("{-", "-}")),
312    ),
313    lang(
314        "dart",
315        "Dart",
316        (0x00, 0xb4, 0xab),
317        &["//"],
318        Some(("/*", "*/")),
319    ),
320    lang(
321        "powershell",
322        "PowerShell",
323        (0x01, 0x24, 0x56),
324        &["#"],
325        Some(("<#", "#>")),
326    ),
327    lang(
328        "svelte",
329        "Svelte",
330        (0xff, 0x3e, 0x00),
331        &[],
332        Some(("<!--", "-->")),
333    ),
334    // no bundled grammar; EXTRA_* routes paths to these
335    lang(
336        "kotlin",
337        "Kotlin",
338        (0xa9, 0x7b, 0xff),
339        &["//"],
340        Some(("/*", "*/")),
341    ),
342    lang("vue", "Vue", (0x41, 0xb8, 0x83), &[], Some(("<!--", "-->"))),
343    lang("r", "R", (0x19, 0x8c, 0xe7), &["#"], None),
344    lang("perl", "Perl", (0x02, 0x98, 0xc3), &["#"], None),
345    lang(
346        "protobuf",
347        "Protocol Buffer",
348        (0xe3, 0xc5, 0x8e),
349        &["//"],
350        Some(("/*", "*/")),
351    ),
352    lang("ini", "INI", (0xd1, 0xdb, 0xe0), &[";", "#"], None),
353    lang("csv", "CSV", (0x23, 0x73, 0x46), &[], None),
354    lang(
355        "scheme",
356        "Scheme",
357        (0x1e, 0x4a, 0xec),
358        &[";"],
359        Some(("#|", "|#")),
360    ),
361    lang("text", "Text", (0x8b, 0x94, 0x9e), &[], None),
362];
363
364const EXTRA_EXTENSIONS: &[(&str, &str)] = &[
365    ("kt", "kotlin"),
366    ("kts", "kotlin"),
367    ("vue", "vue"),
368    ("r", "r"),
369    ("pl", "perl"),
370    ("pm", "perl"),
371    ("proto", "protobuf"),
372    ("ini", "ini"),
373    ("cfg", "ini"),
374    ("conf", "ini"),
375    ("properties", "ini"),
376    ("csv", "csv"),
377    ("tsv", "csv"),
378    ("txt", "text"),
379    ("scm", "scheme"),
380    ("ss", "scheme"),
381];
382
383const EXTRA_FILENAMES: &[(&str, &str)] = &[
384    (".gitignore", "ini"),
385    (".gitattributes", "ini"),
386    (".editorconfig", "ini"),
387];
388
389const fn lang(
390    key: &'static str,
391    name: &'static str,
392    color: Rgb,
393    line_comments: &'static [&'static str],
394    block_comment: Option<(&'static str, &'static str)>,
395) -> (&'static str, Language) {
396    (
397        key,
398        Language {
399            name,
400            color,
401            line_comments,
402            block_comment,
403        },
404    )
405}
406
407#[cfg(test)]
408mod tests {
409    use super::*;
410
411    #[test]
412    fn extensions_resolve_through_the_highlighters_own_table() {
413        assert_eq!(of_path("src/main.rs").map(|l| l.name), Some("Rust"));
414        assert_eq!(of_path("a/b/setup.py").map(|l| l.name), Some("Python"));
415        assert_eq!(of_path("scripts/release.sh").map(|l| l.name), Some("Shell"));
416        assert_eq!(of_path("infra/main.tf").map(|l| l.name), Some("HCL"));
417        assert_eq!(of_path("Makefile").map(|l| l.name), Some("Makefile"));
418    }
419
420    #[test]
421    fn a_license_is_prose_whatever_it_is_suffixed_with() {
422        assert_eq!(of_path("LICENSE-APACHE").map(|l| l.name), Some("Text"));
423        assert_eq!(of_path("LICENSE-MIT").map(|l| l.name), Some("Text"));
424        assert_eq!(of_path("LICENSE.md").map(|l| l.name), Some("Markdown"));
425    }
426
427    #[test]
428    fn a_python_docstring_counts_as_a_comment() {
429        let python = of_path("x.py").expect("python");
430        let source = "\"\"\"What this does.\n\nAnd why.\n\"\"\"\nimport os\n";
431        assert_eq!(count_lines(source, Some(python)), (1, 3, 1));
432        let inline = "def f():\n    \"\"\"Note.\"\"\"\n    return 1\n";
433        assert_eq!(count_lines(inline, Some(python)), (2, 1, 0));
434    }
435
436    #[test]
437    fn languages_without_a_grammar_still_resolve() {
438        assert_eq!(of_path("app/Main.kt").map(|l| l.name), Some("Kotlin"));
439        assert_eq!(
440            of_path("api/user.proto").map(|l| l.name),
441            Some("Protocol Buffer")
442        );
443        assert_eq!(of_path("LICENSE").map(|l| l.name), Some("Text"));
444        assert_eq!(of_path("deploy/app.ini").map(|l| l.name), Some("INI"));
445    }
446
447    #[test]
448    fn an_unknown_path_has_no_language() {
449        assert_eq!(of_path("data.bin"), None);
450        assert_eq!(of_path("no-extension-here"), None);
451    }
452
453    #[test]
454    fn counting_splits_code_comments_and_blanks() {
455        let rust = of_path("x.rs").expect("rust");
456        let source = "fn main() {\n\n    // why\n    let x = 1; // trailing\n}\n";
457        assert_eq!(count_lines(source, Some(rust)), (3, 1, 1));
458    }
459
460    #[test]
461    fn a_block_comment_runs_until_its_end_token() {
462        let rust = of_path("x.rs").expect("rust");
463        let source = "/* one\n   two\n   three */\nfn main() {}\n";
464        assert_eq!(count_lines(source, Some(rust)), (1, 3, 0));
465    }
466
467    #[test]
468    fn a_block_that_opens_and_closes_on_one_line_leaves_nothing_open() {
469        let rust = of_path("x.rs").expect("rust");
470        let source = "/* note */\nfn main() {}\nlet y = 2;\n";
471        assert_eq!(count_lines(source, Some(rust)), (2, 1, 0));
472    }
473
474    #[test]
475    fn a_shebang_counts_as_code_even_where_its_token_opens_comments() {
476        let shell = of_path("run.sh").expect("shell");
477        assert_eq!(
478            count_lines("#!/bin/sh\n# note\necho hi\n", Some(shell)),
479            (2, 1, 0)
480        );
481    }
482
483    #[test]
484    fn a_block_opener_that_starts_with_the_line_token_still_opens_a_block() {
485        let lua = of_path("init.lua").expect("lua");
486        let source = "--[[\n  long note\n  more\n]]\nprint(1)\n";
487        assert_eq!(count_lines(source, Some(lua)), (1, 4, 0));
488        assert_eq!(count_lines("-- note\nprint(1)\n", Some(lua)), (1, 1, 0));
489    }
490
491    #[test]
492    fn an_unknown_language_counts_every_filled_line_as_code() {
493        assert_eq!(count_lines("a\n\nb\n", None), (2, 0, 1));
494    }
495
496    #[test]
497    fn a_dark_ground_lifts_the_colours_that_would_vanish_on_it() {
498        let terminal = (0x0d, 0x11, 0x17);
499        let json = of_path("a.json").expect("json").color;
500        assert_eq!(json, (0x29, 0x29, 0x29), "the table keeps Linguist's hex");
501        let lifted = readable_on(json, terminal);
502        assert!(
503            luminance(lifted) > luminance(json),
504            "JSON's near-black lifts off the terminal ground: {lifted:?}"
505        );
506        assert!(contrast(lifted, terminal) >= 3.0);
507    }
508
509    #[test]
510    fn a_colour_that_already_reads_is_left_alone() {
511        let terminal = (0x0d, 0x11, 0x17);
512        let rust = of_path("a.rs").expect("rust").color;
513        assert_eq!(readable_on(rust, terminal), rust);
514    }
515
516    #[test]
517    fn a_light_ground_darkens_instead_of_lifting() {
518        let page = (0xff, 0xff, 0xff);
519        let shell = of_path("a.sh").expect("shell").color;
520        let fitted = readable_on(shell, page);
521        assert!(
522            luminance(fitted) < luminance(shell),
523            "Linguist's bright green has to darken for a light theme: {fitted:?}"
524        );
525        assert!(contrast(fitted, page) >= 3.0);
526    }
527}
diffler_core/language.rs

diffler_core/
language.rs
