comex 1.0.0

Fast, dependency-free comment extraction and removal for source files, with content-based syntax detection (no file extensions required).
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
//! Comment-syntax families ("dialects") and the rule tables that describe them.
//!
//! A [`Syntax`] is not a language: it is a *set of lexical rules* shared by many
//! languages. `//` + `/* */` + backslash-escaped strings covers C, C++, C#, Java,
//! JavaScript, TypeScript, Go, Rust, Swift, Kotlin, Scala, Dart and PHP; `#` covers
//! Python, Ruby, shell, Perl, YAML, TOML, R, Makefiles and Dockerfiles. Grouping by
//! lexical shape is what makes extension-free detection tractable: there are a few
//! dozen languages worth caring about but only a handful of comment syntaxes.

/// A comment that runs from a marker to the end of the line.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct LineRule {
    /// Literal marker, e.g. `//`.
    pub marker: &'static str,
    /// Only recognised when nothing but whitespace precedes it on the line.
    ///
    /// Used for markers that are ambiguous mid-line (`REM`, `::`, `=begin`).
    pub anchored: bool,
    /// Marker is matched case-insensitively (ASCII), e.g. `REM`/`rem`.
    pub ci: bool,
    /// Bytes that cancel the marker when they immediately precede it.
    ///
    /// `\%` is an escaped percent in TeX and `\;` a character literal in Clojure. For
    /// `//` the guard is `:`, which is what makes `https://example.com` survive even
    /// when a file is scanned with the wrong syntax - by far the most common way a
    /// comment stripper corrupts real text.
    pub not_after: &'static [u8],
    /// Marker must start a word: at the start of a line, or after whitespace.
    ///
    /// This is what POSIX shell, YAML, TOML and Ruby actually require of `#`, and it is
    /// the difference between a comment and the fragment in
    /// `url: https://example.com/#top`.
    pub boundary_before: bool,
    /// Marker must be followed by end-of-line or one of these bytes.
    ///
    /// Guards word-shaped markers so `REMOVE` is not read as `REM` + `OVE`.
    pub require_after: Option<&'static [u8]>,
}

impl LineRule {
    const fn new(marker: &'static str) -> Self {
        LineRule {
            marker,
            anchored: false,
            ci: false,
            not_after: &[],
            boundary_before: false,
            require_after: None,
        }
    }
    const fn anchored(mut self) -> Self {
        self.anchored = true;
        self
    }
    const fn word(mut self) -> Self {
        self.ci = true;
        self.require_after = Some(b" \t\r");
        self
    }
    const fn not_after(mut self, bytes: &'static [u8]) -> Self {
        self.not_after = bytes;
        self
    }
    const fn boundary_before(mut self) -> Self {
        self.boundary_before = true;
        self
    }
}

/// A comment delimited by an opening and closing marker, possibly spanning lines.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BlockRule {
    /// Opening marker, e.g. `/*`.
    pub open: &'static str,
    /// Closing marker, e.g. `*/`.
    pub close: &'static str,
    /// Nested opens must be matched by nested closes (Rust, Haskell, OCaml, Scheme).
    pub nestable: bool,
    /// Only recognised at the start of a line (`=begin`/`=end` in Ruby).
    pub anchored: bool,
    /// Closing marker must also sit at the start of a line.
    pub close_anchored: bool,
}

impl BlockRule {
    const fn new(open: &'static str, close: &'static str) -> Self {
        BlockRule {
            open,
            close,
            nestable: false,
            anchored: false,
            close_anchored: false,
        }
    }
    const fn nestable(mut self) -> Self {
        self.nestable = true;
        self
    }
    const fn line_anchored(mut self) -> Self {
        self.anchored = true;
        self.close_anchored = true;
        self
    }
}

/// A string or character literal. Strings are not output, they are *protected*:
/// a `#` inside `"..."` must not be mistaken for a comment.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct StringRule {
    /// Opening delimiter, e.g. `"` or `"""`.
    pub open: &'static str,
    /// Closing delimiter, usually the same as `open`.
    pub close: &'static str,
    /// Escape byte, usually `\`. `None` means no escape processing.
    pub escape: Option<u8>,
    /// Literal may contain raw newlines. When false an unterminated literal ends at
    /// the newline, which bounds the damage a stray quote can do to one line.
    pub multiline: bool,
    /// A doubled closing marker is an escaped literal marker (`''` in SQL, `""` in VB).
    pub doubling: bool,
    /// Honour `r"..."` / `r#"..."#` style raw-string prefixes (Rust, Python, C#).
    pub raw_prefix: bool,
    /// Treat as a literal only when it closes within a few bytes on the same line.
    ///
    /// This is what keeps Rust lifetimes (`&'a T`), Matlab transposes (`A'`) and
    /// prose apostrophes from opening a string that swallows real comments.
    pub char_like: bool,
    /// May be reported as a documentation comment (Python/Elixir triple quotes).
    pub doc_capable: bool,
}

impl StringRule {
    const fn new(open: &'static str, close: &'static str) -> Self {
        StringRule {
            open,
            close,
            escape: Some(b'\\'),
            multiline: false,
            doubling: false,
            raw_prefix: false,
            char_like: false,
            doc_capable: false,
        }
    }
    const fn quote(q: &'static str) -> Self {
        Self::new(q, q)
    }
    const fn no_escape(mut self) -> Self {
        self.escape = None;
        self
    }
    const fn multiline(mut self) -> Self {
        self.multiline = true;
        self
    }
    const fn doubling(mut self) -> Self {
        self.doubling = true;
        self
    }
    const fn raw_prefix(mut self) -> Self {
        self.raw_prefix = true;
        self
    }
    const fn char_like(mut self) -> Self {
        self.char_like = true;
        self
    }
    const fn doc_capable(mut self) -> Self {
        self.doc_capable = true;
        self
    }
}

/// The comment-syntax families comex understands.
///
/// Each variant names a lexical family, with the common languages it covers.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Dialect {
    /// No comment syntax could be identified; input is passed through untouched.
    Unknown,
    /// `//`, `/* */` — C, C++, C#, Java, JS, TS, Go, Rust, Swift, Kotlin, PHP, Dart.
    CFamily,
    /// `#` — Python, Ruby, shell, Perl, YAML, TOML, R, Make, Dockerfile, PowerShell.
    Hash,
    /// `--`, `/* */` — SQL and friends.
    Sql,
    /// `--`, `--[[ ]]` — Lua.
    Lua,
    /// `--`, `{- -}` — Haskell, Elm, PureScript, Agda.
    Haskell,
    /// `;`, `#| |#` — Lisp, Scheme, Clojure, Emacs Lisp, assembly.
    Lisp,
    /// `<!-- -->` — HTML, XML, SVG, Markdown, Vue, Svelte (with embedded JS/CSS).
    Markup,
    /// `(* *)` — OCaml, F#, Pascal, Mathematica.
    Ml,
    /// `%` — TeX/LaTeX, Erlang, Prolog, Matlab/Octave, PostScript.
    Percent,
    /// `'`, `REM` — Visual Basic, VBA, VBScript.
    VisualBasic,
    /// `REM`, `::` — Windows batch files.
    Batch,
    /// `!` — modern Fortran.
    Fortran,
    /// `;` and `#` — INI, desktop entries, gitconfig-style files.
    Ini,
}

impl Dialect {
    /// Every dialect, in the order the CLI lists them.
    pub const ALL: &'static [Dialect] = &[
        Dialect::CFamily,
        Dialect::Hash,
        Dialect::Sql,
        Dialect::Lua,
        Dialect::Haskell,
        Dialect::Lisp,
        Dialect::Markup,
        Dialect::Ml,
        Dialect::Percent,
        Dialect::VisualBasic,
        Dialect::Batch,
        Dialect::Fortran,
        Dialect::Ini,
        Dialect::Unknown,
    ];

    /// Short lowercase name, as accepted by `--syntax`.
    pub fn name(self) -> &'static str {
        match self {
            Dialect::Unknown => "unknown",
            Dialect::CFamily => "c",
            Dialect::Hash => "hash",
            Dialect::Sql => "sql",
            Dialect::Lua => "lua",
            Dialect::Haskell => "haskell",
            Dialect::Lisp => "lisp",
            Dialect::Markup => "markup",
            Dialect::Ml => "ml",
            Dialect::Percent => "percent",
            Dialect::VisualBasic => "vb",
            Dialect::Batch => "batch",
            Dialect::Fortran => "fortran",
            Dialect::Ini => "ini",
        }
    }

    /// One-line human description, including the markers and typical languages.
    pub fn describe(self) -> &'static str {
        match self {
            Dialect::Unknown => "no comment syntax detected (input passes through unchanged)",
            Dialect::CFamily => "// and /* */ - C, C++, C#, Java, JS, TS, Go, Rust, Swift, PHP",
            Dialect::Hash => "# - Python, Ruby, shell, Perl, YAML, TOML, R, Make, Dockerfile",
            Dialect::Sql => "-- and /* */ - SQL dialects",
            Dialect::Lua => "-- and --[[ ]] - Lua",
            Dialect::Haskell => "-- and {- -} - Haskell, Elm, PureScript",
            Dialect::Lisp => "; and #| |# - Lisp, Scheme, Clojure, assembly",
            Dialect::Markup => "<!-- --> - HTML, XML, SVG, Markdown, Vue, Svelte",
            Dialect::Ml => "(* *) - OCaml, F#, Pascal, Mathematica",
            Dialect::Percent => "% - TeX, Erlang, Prolog, Matlab/Octave",
            Dialect::VisualBasic => "' and REM - Visual Basic, VBA, VBScript",
            Dialect::Batch => "REM and :: - Windows batch",
            Dialect::Fortran => "! - modern Fortran",
            Dialect::Ini => "; and # - INI, desktop entries, gitconfig",
        }
    }

    /// Parse a dialect name (case-insensitive), accepting a few aliases.
    pub fn from_name(s: &str) -> Option<Dialect> {
        let lower = s.to_ascii_lowercase();
        Some(match lower.as_str() {
            "unknown" | "none" | "plain" => Dialect::Unknown,
            "c" | "c-family" | "cfamily" | "clike" | "c-like" => Dialect::CFamily,
            "hash" | "pound" | "shell" | "python" => Dialect::Hash,
            "sql" => Dialect::Sql,
            "lua" => Dialect::Lua,
            "haskell" | "hs" | "elm" => Dialect::Haskell,
            "lisp" | "scheme" | "clojure" | "asm" => Dialect::Lisp,
            "markup" | "html" | "xml" | "md" | "markdown" => Dialect::Markup,
            "ml" | "ocaml" | "fsharp" | "pascal" => Dialect::Ml,
            "percent" | "tex" | "latex" | "erlang" | "matlab" => Dialect::Percent,
            "vb" | "basic" | "vba" | "vbscript" => Dialect::VisualBasic,
            "batch" | "bat" | "cmd" => Dialect::Batch,
            "fortran" | "f90" => Dialect::Fortran,
            "ini" | "conf" | "toml-ini" => Dialect::Ini,
            _ => return None,
        })
    }

    /// The lexical rules for this dialect.
    pub fn syntax(self) -> Syntax {
        Syntax::of(self)
    }
}

// --- Rule tables -----------------------------------------------------------
//
// Within each array, longer markers MUST precede shorter ones that they start
// with (`"""` before `"`), because the scanner takes the first match.

// `\` guards JavaScript regex literals - `/https?:\/\//` ends with an escaped slash
// immediately followed by the closing delimiter. `:` guards URL schemes.
const C_LINES: &[LineRule] = &[LineRule::new("//").not_after(b"\\:")];
const C_BLOCKS: &[BlockRule] = &[BlockRule::new("/*", "*/")];
const C_STRINGS: &[StringRule] = &[
    StringRule::quote("\"\"\"").multiline(), // Java 15+, Kotlin, Scala, Swift
    StringRule::quote("\"").raw_prefix(),
    StringRule::quote("`").multiline(), // JS template literals
    StringRule::quote("'").char_like(), // char literal, not Rust lifetime
];

const HASH_LINES: &[LineRule] = &[LineRule::new("#").boundary_before()];
const HASH_BLOCKS: &[BlockRule] = &[
    BlockRule::new("<#", "#>"),                       // PowerShell
    BlockRule::new("#=", "=#").nestable(),            // Julia
    BlockRule::new("=begin", "=end").line_anchored(), // Ruby
];
const HASH_STRINGS: &[StringRule] = &[
    StringRule::quote("\"\"\"").multiline().doc_capable(),
    StringRule::quote("'''").multiline().doc_capable(),
    StringRule::quote("\"").raw_prefix(),
    StringRule::quote("'"),
    StringRule::quote("`").multiline(), // shell command substitution
];

const SQL_LINES: &[LineRule] = &[LineRule::new("--")];
const SQL_BLOCKS: &[BlockRule] = &[BlockRule::new("/*", "*/").nestable()];
const SQL_STRINGS: &[StringRule] = &[
    StringRule::quote("'").no_escape().doubling().multiline(),
    StringRule::quote("\"").no_escape().doubling(),
    StringRule::quote("`").no_escape(), // MySQL identifiers
];

const LUA_LINES: &[LineRule] = &[LineRule::new("--")];
const LUA_STRINGS: &[StringRule] = &[StringRule::quote("\""), StringRule::quote("'")];

const HASKELL_LINES: &[LineRule] = &[LineRule::new("--")];
const HASKELL_BLOCKS: &[BlockRule] = &[BlockRule::new("{-", "-}").nestable()];
const HASKELL_STRINGS: &[StringRule] =
    &[StringRule::quote("\""), StringRule::quote("'").char_like()];

// `\;` is a Clojure character literal and `#\;` a Scheme one.
const LISP_LINES: &[LineRule] = &[LineRule::new(";").not_after(b"\\")];
const LISP_BLOCKS: &[BlockRule] = &[BlockRule::new("#|", "|#").nestable()];
const LISP_STRINGS: &[StringRule] = &[StringRule::quote("\"").multiline()];

const MARKUP_BLOCKS: &[BlockRule] = &[BlockRule::new("<!--", "-->")];
// Markdown inline code. Prose that mentions `<!-- -->` in backticks is documentation,
// not a comment. HTML rarely has backticks, and an unpaired one only costs its own line.
const MARKUP_STRINGS: &[StringRule] = &[StringRule::quote("`").no_escape()];

const ML_LINES: &[LineRule] = &[LineRule::new("//")]; // F#, Pascal
const ML_BLOCKS: &[BlockRule] = &[BlockRule::new("(*", "*)").nestable()];
const ML_STRINGS: &[StringRule] = &[StringRule::quote("\""), StringRule::quote("'").char_like()];

const PERCENT_LINES: &[LineRule] = &[LineRule::new("%").not_after(b"\\")];
const PERCENT_BLOCKS: &[BlockRule] = &[BlockRule::new("%{", "%}").line_anchored()];
const PERCENT_STRINGS: &[StringRule] = &[StringRule::quote("\"")];

const VB_LINES: &[LineRule] = &[LineRule::new("'"), LineRule::new("REM").word()];
const VB_STRINGS: &[StringRule] = &[StringRule::quote("\"").no_escape().doubling()];

const BATCH_LINES: &[LineRule] = &[
    LineRule::new("REM").word().anchored(),
    LineRule::new("::").anchored(),
];
const BATCH_STRINGS: &[StringRule] = &[StringRule::quote("\"").no_escape()];

const FORTRAN_LINES: &[LineRule] = &[LineRule::new("!")];
const FORTRAN_STRINGS: &[StringRule] = &[
    StringRule::quote("'").no_escape().doubling(),
    StringRule::quote("\"").no_escape().doubling(),
];

const INI_LINES: &[LineRule] = &[
    LineRule::new(";").boundary_before(),
    LineRule::new("#").boundary_before(),
];
const INI_STRINGS: &[StringRule] = &[StringRule::quote("\""), StringRule::quote("'")];

/// The lexical rules used to scan a file.
///
/// Obtain one with [`Dialect::syntax`], [`crate::detect`], or build a custom set.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Syntax {
    /// The family these rules came from.
    pub dialect: Dialect,
    /// To-end-of-line comment markers.
    pub lines: &'static [LineRule],
    /// Delimited comment markers.
    pub blocks: &'static [BlockRule],
    /// String and character literals, tracked so their contents are never mistaken
    /// for comment markers.
    pub strings: &'static [StringRule],
    /// Lua long brackets: `[[ ]]`, `[=[ ]=]` strings and `--[[ ]]` comments.
    pub long_brackets: bool,
    /// Scan `<script>` / `<style>` bodies with C-family rules.
    pub html_embedded: bool,
    /// A leading `#!` line is code, never a comment.
    pub shebang_is_code: bool,
    /// Two-byte prefix introducing a literal character, e.g. `#\` in Scheme.
    pub char_prefix: Option<&'static str>,
}

impl Syntax {
    /// Rules for a dialect.
    pub const fn of(dialect: Dialect) -> Syntax {
        let base = Syntax {
            dialect,
            lines: &[],
            blocks: &[],
            strings: &[],
            long_brackets: false,
            html_embedded: false,
            shebang_is_code: true,
            char_prefix: None,
        };
        match dialect {
            Dialect::Unknown => base,
            Dialect::CFamily => Syntax {
                lines: C_LINES,
                blocks: C_BLOCKS,
                strings: C_STRINGS,
                ..base
            },
            Dialect::Hash => Syntax {
                lines: HASH_LINES,
                blocks: HASH_BLOCKS,
                strings: HASH_STRINGS,
                ..base
            },
            Dialect::Sql => Syntax {
                lines: SQL_LINES,
                blocks: SQL_BLOCKS,
                strings: SQL_STRINGS,
                ..base
            },
            Dialect::Lua => Syntax {
                lines: LUA_LINES,
                strings: LUA_STRINGS,
                long_brackets: true,
                ..base
            },
            Dialect::Haskell => Syntax {
                lines: HASKELL_LINES,
                blocks: HASKELL_BLOCKS,
                strings: HASKELL_STRINGS,
                ..base
            },
            Dialect::Lisp => Syntax {
                lines: LISP_LINES,
                blocks: LISP_BLOCKS,
                strings: LISP_STRINGS,
                char_prefix: Some("#\\"),
                ..base
            },
            Dialect::Markup => Syntax {
                blocks: MARKUP_BLOCKS,
                strings: MARKUP_STRINGS,
                html_embedded: true,
                ..base
            },
            Dialect::Ml => Syntax {
                lines: ML_LINES,
                blocks: ML_BLOCKS,
                strings: ML_STRINGS,
                ..base
            },
            Dialect::Percent => Syntax {
                lines: PERCENT_LINES,
                blocks: PERCENT_BLOCKS,
                strings: PERCENT_STRINGS,
                ..base
            },
            Dialect::VisualBasic => Syntax {
                lines: VB_LINES,
                strings: VB_STRINGS,
                ..base
            },
            Dialect::Batch => Syntax {
                lines: BATCH_LINES,
                strings: BATCH_STRINGS,
                ..base
            },
            Dialect::Fortran => Syntax {
                lines: FORTRAN_LINES,
                strings: FORTRAN_STRINGS,
                ..base
            },
            Dialect::Ini => Syntax {
                lines: INI_LINES,
                strings: INI_STRINGS,
                ..base
            },
        }
    }

    /// True when this syntax recognises no comments at all.
    pub fn is_empty(&self) -> bool {
        self.lines.is_empty() && self.blocks.is_empty() && !self.long_brackets
    }

    /// Lookup table of bytes that can begin any rule, so the scanner's hot loop is a
    /// single table probe per byte.
    pub(crate) fn interesting(&self) -> [bool; 256] {
        let mut t = [false; 256];
        let mut mark = |s: &str, ci: bool| {
            if let Some(&b) = s.as_bytes().first() {
                t[b as usize] = true;
                if ci {
                    t[b.to_ascii_lowercase() as usize] = true;
                    t[b.to_ascii_uppercase() as usize] = true;
                }
            }
        };
        for r in self.lines {
            mark(r.marker, r.ci);
        }
        for r in self.blocks {
            mark(r.open, false);
        }
        for r in self.strings {
            // Raw prefixes (`r"`, `br"`, `r#"`) are found by looking *backwards* from
            // the quote, so only the quote byte itself needs to be interesting.
            mark(r.open, false);
        }
        if let Some(p) = self.char_prefix {
            mark(p, false);
        }
        if self.long_brackets {
            t[b'[' as usize] = true;
        }
        if self.html_embedded {
            t[b'<' as usize] = true;
        }
        t
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn every_dialect_round_trips_through_its_name() {
        for &d in Dialect::ALL {
            assert_eq!(Dialect::from_name(d.name()), Some(d), "{}", d.name());
        }
    }

    #[test]
    fn longer_markers_are_ordered_before_their_prefixes() {
        for &d in Dialect::ALL {
            let syn = d.syntax();
            for (i, a) in syn.strings.iter().enumerate() {
                for b in &syn.strings[i + 1..] {
                    assert!(
                        !b.open.starts_with(a.open) || a.open == b.open,
                        "{}: {:?} shadows {:?}",
                        d.name(),
                        a.open,
                        b.open
                    );
                }
            }
        }
    }

    #[test]
    fn unknown_syntax_recognises_nothing() {
        assert!(Dialect::Unknown.syntax().is_empty());
        assert!(!Dialect::CFamily.syntax().is_empty());
    }
}