diffler_core/
language.rs

1//! What language a path is written in, the colour GitHub paints it, and the
2//! comment syntax the line counter needs.
3//!
4//! Detection is a pure path function: the syntax registry already maps every
5//! extension it can highlight to a grammar, so that mapping stays the one
6//! source of truth and this module adds the languages diffler counts but does
7//! not highlight. Colours are Linguist's own hexes, the ones a reader knows
8//! from a repository page, lifted toward the foreground when the terminal's
9//! background would swallow them.
10
11use crate::syntax::registry::REGISTRY;
12
13pub type Rgb = (u8, u8, u8);
14
15/// A language diffler can name, with everything the breakdown needs about it.
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17pub struct Language {
18    /// Display label, spelled the way a repository page spells it.
19    pub name: &'static str,
20    /// Linguist's colour for the language, untouched.
21    pub color: Rgb,
22    line_comments: &'static [&'static str],
23    block_comment: Option<(&'static str, &'static str)>,
24}
25
26impl Language {
27    /// Whether `line`, already trimmed, opens a comment that ends on the same
28    /// line, and what remains open after it.
29    fn classify(self, line: &str, in_block: bool) -> (LineKind, bool) {
30        if in_block {
31            let closed = self
32                .block_comment
33                .is_some_and(|(_, end)| line.contains(end));
34            return (LineKind::Comment, !closed);
35        }
36        // the block opener is tested first because it can start with the line
37        // token itself: Lua's `--[[` opens a block, `--` only a line
38        if let Some((start, end)) = self.block_comment
39            && line.starts_with(start)
40        {
41            let closed = line[start.len()..].contains(end);
42            return (LineKind::Comment, !closed);
43        }
44        if self
45            .line_comments
46            .iter()
47            .any(|token| line.starts_with(token))
48        {
49            return (LineKind::Comment, false);
50        }
51        (LineKind::Code, false)
52    }
53}
54
55enum LineKind {
56    Code,
57    Comment,
58}
59
60/// The `(code, comments, blanks)` a source text holds. A line counts as a
61/// comment when it opens with one, so a trailing `// note` after code reads as
62/// code, the same call `scc` and `cloc` make.
63#[must_use]
64pub fn count_lines(text: &str, language: Option<Language>) -> (usize, usize, usize) {
65    let (mut code, mut comments, mut blanks) = (0, 0, 0);
66    let mut in_block = false;
67    for (index, raw) in text.lines().enumerate() {
68        let line = raw.trim();
69        if line.is_empty() {
70            blanks += 1;
71            continue;
72        }
73        // a shebang is the file's first instruction, and `#` would otherwise
74        // swallow it
75        if index == 0 && line.starts_with("#!") {
76            code += 1;
77            continue;
78        }
79        let Some(language) = language else {
80            code += 1;
81            continue;
82        };
83        let (kind, still_open) = language.classify(line, in_block);
84        in_block = still_open;
85        match kind {
86            LineKind::Code => code += 1,
87            LineKind::Comment => comments += 1,
88        }
89    }
90    (code, comments, blanks)
91}
92
93/// The language of `path`, by extension or by whole filename.
94#[must_use]
95pub fn of_path(path: &str) -> Option<Language> {
96    // the highlighter's registry owns the extension table for everything it can
97    // parse; only what it cannot appears in EXTRA below
98    if let Some(entry) = REGISTRY.for_path(path)
99        && let Some(language) = by_key(entry.name)
100    {
101        return Some(language);
102    }
103    let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase();
104    if let Some(key) = EXTRA_FILENAMES
105        .iter()
106        .find_map(|(filename, key)| (*filename == name).then_some(key))
107    {
108        return by_key(key);
109    }
110    if let Some(extension) = name.rsplit_once('.').map(|(_, ext)| ext)
111        && let Some(key) = EXTRA_EXTENSIONS
112            .iter()
113            .find_map(|(ext, key)| (*ext == extension).then_some(key))
114    {
115        return by_key(key);
116    }
117    // consulted after the extensions so `LICENSE.md` stays Markdown: a repo
118    // ships `LICENSE-APACHE` beside `LICENSE-MIT`, and both are prose
119    if crate::classify::DOC_NAMES
120        .iter()
121        .any(|stem| name.starts_with(stem))
122    {
123        return by_key("text");
124    }
125    None
126}
127
128fn by_key(key: &str) -> Option<Language> {
129    TABLE
130        .iter()
131        .find_map(|(name, language)| (*name == key).then_some(*language))
132}
133
134/// Lift `color` until it separates from `bg`. Linguist's palette is tuned for
135/// a white page, so a few entries (JSON's `#292929`, C's `#555555`) vanish on a
136/// dark terminal and a few of the bright ones wash out on a light one.
137#[must_use]
138pub fn readable_on(color: Rgb, bg: Rgb) -> Rgb {
139    const TARGET: f32 = 3.0;
140    let toward = if luminance(bg) > 0.5 { 0 } else { 255 };
141    let mut out = color;
142    // each step moves a tenth of what is left, and never by less than one
143    // channel value, so this reaches the end of the ramp and stops
144    for _ in 0..40 {
145        if contrast(out, bg) >= TARGET {
146            break;
147        }
148        out = step_toward(out, toward);
149    }
150    out
151}
152
153fn step_toward(color: Rgb, toward: u8) -> Rgb {
154    let blend = |channel: u8| {
155        let (from, to) = (i32::from(channel), i32::from(toward));
156        let by = (to - from) / 10;
157        let stepped = from + if by == 0 { (to - from).signum() } else { by };
158        u8::try_from(stepped.clamp(0, 255)).unwrap_or(toward)
159    };
160    (blend(color.0), blend(color.1), blend(color.2))
161}
162
163fn contrast(a: Rgb, b: Rgb) -> f32 {
164    let (high, low) = {
165        let (la, lb) = (luminance(a), luminance(b));
166        if la > lb { (la, lb) } else { (lb, la) }
167    };
168    (high + 0.05) / (low + 0.05)
169}
170
171/// WCAG relative luminance.
172fn luminance(color: Rgb) -> f32 {
173    let channel = |value: u8| {
174        let value = f32::from(value) / 255.0;
175        if value <= 0.03928 {
176            value / 12.92
177        } else {
178            ((value + 0.055) / 1.055).powf(2.4)
179        }
180    };
181    0.2126 * channel(color.0) + 0.7152 * channel(color.1) + 0.0722 * channel(color.2)
182}
183
184/// Everything the sidebar's grammar names map to, plus the languages diffler
185/// counts without highlighting. Keys match `syntax::registry` names where a
186/// grammar exists; colours are Linguist's.
187const TABLE: &[(&str, Language)] = &[
188    lang(
189        "rust",
190        "Rust",
191        (0xde, 0xa5, 0x84),
192        &["//"],
193        Some(("/*", "*/")),
194    ),
195    lang(
196        "python",
197        "Python",
198        (0x35, 0x72, 0xa5),
199        &["#"],
200        // a docstring is the module's or function's comment, and both fences
201        // are the same token, which `classify` handles by looking past the
202        // opening one
203        Some(("\"\"\"", "\"\"\"")),
204    ),
205    lang(
206        "javascript",
207        "JavaScript",
208        (0xf1, 0xe0, 0x5a),
209        &["//"],
210        Some(("/*", "*/")),
211    ),
212    lang(
213        "typescript",
214        "TypeScript",
215        (0x31, 0x78, 0xc6),
216        &["//"],
217        Some(("/*", "*/")),
218    ),
219    lang(
220        "tsx",
221        "TSX",
222        (0x31, 0x78, 0xc6),
223        &["//"],
224        Some(("/*", "*/")),
225    ),
226    lang("go", "Go", (0x00, 0xad, 0xd8), &["//"], Some(("/*", "*/"))),
227    lang("c", "C", (0x55, 0x55, 0x55), &["//"], Some(("/*", "*/"))),
228    lang(
229        "cpp",
230        "C++",
231        (0xf3, 0x4b, 0x7d),
232        &["//"],
233        Some(("/*", "*/")),
234    ),
235    lang(
236        "java",
237        "Java",
238        (0xb0, 0x72, 0x19),
239        &["//"],
240        Some(("/*", "*/")),
241    ),
242    lang(
243        "c-sharp",
244        "C#",
245        (0x17, 0x86, 0x00),
246        &["//"],
247        Some(("/*", "*/")),
248    ),
249    lang(
250        "ruby",
251        "Ruby",
252        (0x70, 0x15, 0x16),
253        &["#"],
254        Some(("=begin", "=end")),
255    ),
256    lang(
257        "php",
258        "PHP",
259        (0x4f, 0x5d, 0x95),
260        &["//", "#"],
261        Some(("/*", "*/")),
262    ),
263    lang("bash", "Shell", (0x89, 0xe0, 0x51), &["#"], None),
264    lang("json", "JSON", (0x29, 0x29, 0x29), &[], None),
265    lang(
266        "html",
267        "HTML",
268        (0xe3, 0x4c, 0x26),
269        &[],
270        Some(("<!--", "-->")),
271    ),
272    lang("css", "CSS", (0x66, 0x33, 0x99), &[], Some(("/*", "*/"))),
273    lang("yaml", "YAML", (0xcb, 0x17, 0x1e), &["#"], None),
274    lang(
275        "sql",
276        "SQL",
277        (0xe3, 0x8c, 0x00),
278        &["--"],
279        Some(("/*", "*/")),
280    ),
281    lang(
282        "markdown",
283        "Markdown",
284        (0x08, 0x3f, 0xa1),
285        &[],
286        Some(("<!--", "-->")),
287    ),
288    lang("toml", "TOML", (0x9c, 0x42, 0x21), &["#"], None),
289    lang(
290        "hcl",
291        "HCL",
292        (0x84, 0x4f, 0xba),
293        &["#", "//"],
294        Some(("/*", "*/")),
295    ),
296    lang("dockerfile", "Dockerfile", (0x38, 0x4d, 0x54), &["#"], None),
297    lang("make", "Makefile", (0x42, 0x78, 0x19), &["#"], None),
298    lang(
299        "lua",
300        "Lua",
301        (0x00, 0x00, 0x80),
302        &["--"],
303        Some(("--[[", "]]")),
304    ),
305    lang("nix", "Nix", (0x7e, 0x7e, 0xff), &["#"], Some(("/*", "*/"))),
306    lang("xml", "XML", (0x00, 0x60, 0xac), &[], Some(("<!--", "-->"))),
307    lang(
308        "swift",
309        "Swift",
310        (0xf0, 0x51, 0x38),
311        &["//"],
312        Some(("/*", "*/")),
313    ),
314    lang(
315        "scala",
316        "Scala",
317        (0xc2, 0x2d, 0x40),
318        &["//"],
319        Some(("/*", "*/")),
320    ),
321    lang(
322        "elixir",
323        "Elixir",
324        (0x6e, 0x4a, 0x7e),
325        &["#"],
326        Some(("@doc \"\"\"", "\"\"\"")),
327    ),
328    lang("zig", "Zig", (0xec, 0x91, 0x5c), &["//"], None),
329    lang(
330        "haskell",
331        "Haskell",
332        (0x5e, 0x50, 0x86),
333        &["--"],
334        Some(("{-", "-}")),
335    ),
336    lang(
337        "dart",
338        "Dart",
339        (0x00, 0xb4, 0xab),
340        &["//"],
341        Some(("/*", "*/")),
342    ),
343    lang(
344        "powershell",
345        "PowerShell",
346        (0x01, 0x24, 0x56),
347        &["#"],
348        Some(("<#", "#>")),
349    ),
350    lang(
351        "svelte",
352        "Svelte",
353        (0xff, 0x3e, 0x00),
354        &[],
355        Some(("<!--", "-->")),
356    ),
357    // no grammar bundled for these; EXTRA_* below routes paths to them
358    lang(
359        "kotlin",
360        "Kotlin",
361        (0xa9, 0x7b, 0xff),
362        &["//"],
363        Some(("/*", "*/")),
364    ),
365    lang("vue", "Vue", (0x41, 0xb8, 0x83), &[], Some(("<!--", "-->"))),
366    lang("r", "R", (0x19, 0x8c, 0xe7), &["#"], None),
367    lang("perl", "Perl", (0x02, 0x98, 0xc3), &["#"], None),
368    lang(
369        "protobuf",
370        "Protocol Buffer",
371        (0xe3, 0xc5, 0x8e),
372        &["//"],
373        Some(("/*", "*/")),
374    ),
375    lang("ini", "INI", (0xd1, 0xdb, 0xe0), &[";", "#"], None),
376    lang("csv", "CSV", (0x23, 0x73, 0x46), &[], None),
377    lang(
378        "scheme",
379        "Scheme",
380        (0x1e, 0x4a, 0xec),
381        &[";"],
382        Some(("#|", "|#")),
383    ),
384    lang("text", "Text", (0x8b, 0x94, 0x9e), &[], None),
385];
386
387/// Extensions the highlighter has no grammar for.
388const EXTRA_EXTENSIONS: &[(&str, &str)] = &[
389    ("kt", "kotlin"),
390    ("kts", "kotlin"),
391    ("vue", "vue"),
392    ("r", "r"),
393    ("pl", "perl"),
394    ("pm", "perl"),
395    ("proto", "protobuf"),
396    ("ini", "ini"),
397    ("cfg", "ini"),
398    ("conf", "ini"),
399    ("properties", "ini"),
400    ("csv", "csv"),
401    ("tsv", "csv"),
402    ("txt", "text"),
403    ("scm", "scheme"),
404    ("ss", "scheme"),
405];
406
407const EXTRA_FILENAMES: &[(&str, &str)] = &[
408    (".gitignore", "ini"),
409    (".gitattributes", "ini"),
410    (".editorconfig", "ini"),
411];
412
413const fn lang(
414    key: &'static str,
415    name: &'static str,
416    color: Rgb,
417    line_comments: &'static [&'static str],
418    block_comment: Option<(&'static str, &'static str)>,
419) -> (&'static str, Language) {
420    (
421        key,
422        Language {
423            name,
424            color,
425            line_comments,
426            block_comment,
427        },
428    )
429}
430
431#[cfg(test)]
432mod tests {
433    use super::*;
434
435    #[test]
436    fn extensions_resolve_through_the_highlighters_own_table() {
437        assert_eq!(of_path("src/main.rs").map(|l| l.name), Some("Rust"));
438        assert_eq!(of_path("a/b/setup.py").map(|l| l.name), Some("Python"));
439        // the registry maps sh, bash and zsh to one grammar; all read as Shell
440        assert_eq!(of_path("scripts/release.sh").map(|l| l.name), Some("Shell"));
441        assert_eq!(of_path("infra/main.tf").map(|l| l.name), Some("HCL"));
442        assert_eq!(of_path("Makefile").map(|l| l.name), Some("Makefile"));
443    }
444
445    #[test]
446    fn a_license_is_prose_whatever_it_is_suffixed_with() {
447        assert_eq!(of_path("LICENSE-APACHE").map(|l| l.name), Some("Text"));
448        assert_eq!(of_path("LICENSE-MIT").map(|l| l.name), Some("Text"));
449        // the extension still wins, so a markdown licence stays markdown
450        assert_eq!(of_path("LICENSE.md").map(|l| l.name), Some("Markdown"));
451    }
452
453    #[test]
454    fn a_python_docstring_counts_as_a_comment() {
455        let python = of_path("x.py").expect("python");
456        let source = "\"\"\"What this does.\n\nAnd why.\n\"\"\"\nimport os\n";
457        assert_eq!(count_lines(source, Some(python)), (1, 3, 1));
458        // one line holding both fences opens nothing
459        let inline = "def f():\n    \"\"\"Note.\"\"\"\n    return 1\n";
460        assert_eq!(count_lines(inline, Some(python)), (2, 1, 0));
461    }
462
463    #[test]
464    fn languages_without_a_grammar_still_resolve() {
465        assert_eq!(of_path("app/Main.kt").map(|l| l.name), Some("Kotlin"));
466        assert_eq!(
467            of_path("api/user.proto").map(|l| l.name),
468            Some("Protocol Buffer")
469        );
470        assert_eq!(of_path("LICENSE").map(|l| l.name), Some("Text"));
471        assert_eq!(of_path("deploy/app.ini").map(|l| l.name), Some("INI"));
472    }
473
474    #[test]
475    fn an_unknown_path_has_no_language() {
476        assert_eq!(of_path("data.bin"), None);
477        assert_eq!(of_path("no-extension-here"), None);
478    }
479
480    #[test]
481    fn counting_splits_code_comments_and_blanks() {
482        let rust = of_path("x.rs").expect("rust");
483        let source = "fn main() {\n\n    // why\n    let x = 1; // trailing\n}\n";
484        assert_eq!(count_lines(source, Some(rust)), (3, 1, 1));
485    }
486
487    #[test]
488    fn a_block_comment_runs_until_its_end_token() {
489        let rust = of_path("x.rs").expect("rust");
490        let source = "/* one\n   two\n   three */\nfn main() {}\n";
491        assert_eq!(count_lines(source, Some(rust)), (1, 3, 0));
492    }
493
494    #[test]
495    fn a_block_that_opens_and_closes_on_one_line_leaves_nothing_open() {
496        let rust = of_path("x.rs").expect("rust");
497        let source = "/* note */\nfn main() {}\nlet y = 2;\n";
498        assert_eq!(count_lines(source, Some(rust)), (2, 1, 0));
499    }
500
501    #[test]
502    fn a_shebang_counts_as_code_even_where_its_token_opens_comments() {
503        let shell = of_path("run.sh").expect("shell");
504        assert_eq!(
505            count_lines("#!/bin/sh\n# note\necho hi\n", Some(shell)),
506            (2, 1, 0)
507        );
508    }
509
510    /// Lua opens a block with `--[[` and a line with `--`, so the longer token
511    /// has to be tried first or every block reads as one line comment.
512    #[test]
513    fn a_block_opener_that_starts_with_the_line_token_still_opens_a_block() {
514        let lua = of_path("init.lua").expect("lua");
515        let source = "--[[\n  long note\n  more\n]]\nprint(1)\n";
516        assert_eq!(count_lines(source, Some(lua)), (1, 4, 0));
517        // a plain line comment still ends at its own line
518        assert_eq!(count_lines("-- note\nprint(1)\n", Some(lua)), (1, 1, 0));
519    }
520
521    #[test]
522    fn an_unknown_language_counts_every_filled_line_as_code() {
523        assert_eq!(count_lines("a\n\nb\n", None), (2, 0, 1));
524    }
525
526    #[test]
527    fn a_dark_ground_lifts_the_colours_that_would_vanish_on_it() {
528        let terminal = (0x0d, 0x11, 0x17);
529        let json = of_path("a.json").expect("json").color;
530        assert_eq!(json, (0x29, 0x29, 0x29), "the table keeps Linguist's hex");
531        let lifted = readable_on(json, terminal);
532        assert!(
533            luminance(lifted) > luminance(json),
534            "JSON's near-black lifts off the terminal ground: {lifted:?}"
535        );
536        assert!(contrast(lifted, terminal) >= 3.0);
537    }
538
539    #[test]
540    fn a_colour_that_already_reads_is_left_alone() {
541        let terminal = (0x0d, 0x11, 0x17);
542        let rust = of_path("a.rs").expect("rust").color;
543        assert_eq!(readable_on(rust, terminal), rust);
544    }
545
546    #[test]
547    fn a_light_ground_darkens_instead_of_lifting() {
548        let page = (0xff, 0xff, 0xff);
549        let shell = of_path("a.sh").expect("shell").color;
550        let fitted = readable_on(shell, page);
551        assert!(
552            luminance(fitted) < luminance(shell),
553            "Linguist's bright green has to darken for a light theme: {fitted:?}"
554        );
555        assert!(contrast(fitted, page) >= 3.0);
556    }
557}
diffler_core/language.rs

diffler_core/
language.rs
