fdu_core/content/
content_code_metrics.rs

1//! Streaming common-language source-line classification.
2
3use super::MetricValues;
4
5/// Streaming `code-sloc-v1` counter for a supported language.
6///
7/// The counter retains the current logical line, its allocated capacity, and parser
8/// state. A one-line minified or generated source can therefore require file-sized
9/// memory per active worker. Mixed
10/// code/comment lines are code, blank lines inside block comments are comments, and
11/// multiline string lines are code. Line endings follow the same LF, CRLF, lone-CR,
12/// and unterminated-final-line contract as the basic analyzer.
13#[derive(Debug)]
14pub struct CodeAccumulator {
15    syntax: Syntax,
16    state: State,
17    line: Vec<u8>,
18    previous_cr: bool,
19    metrics: MetricValues,
20}
21
22impl CodeAccumulator {
23    /// Create a counter for one stable file-type ID, or return `None` when
24    /// `code-sloc-v1` does not claim that language.
25    pub fn for_type(file_type: &str) -> Option<Self> {
26        let syntax = Syntax::for_type(file_type)?;
27        Some(Self {
28            syntax,
29            state: State::Normal,
30            line: Vec::new(),
31            previous_cr: false,
32            metrics: MetricValues::default(),
33        })
34    }
35
36    /// Consume an arbitrary byte chunk.
37    pub fn push(&mut self, chunk: &[u8]) {
38        for &byte in chunk {
39            if self.previous_cr {
40                self.previous_cr = false;
41                if byte == b'\n' {
42                    continue;
43                }
44            }
45            match byte {
46                b'\r' => {
47                    self.finish_line();
48                    self.previous_cr = true;
49                }
50                b'\n' => self.finish_line(),
51                _ => self.line.push(byte),
52            }
53        }
54    }
55
56    /// Finish an unterminated final line and return its additive metrics.
57    pub fn finish(mut self) -> MetricValues {
58        if !self.line.is_empty() && self.line.as_slice() != [0xef, 0xbb, 0xbf] {
59            self.finish_line();
60        }
61        self.metrics
62    }
63
64    fn finish_line(&mut self) {
65        let class = classify_line(self.syntax, &mut self.state, &self.line);
66        self.metrics.physical_lines = self.metrics.physical_lines.saturating_add(1);
67        match class {
68            LineClass::Code => {
69                self.metrics.code_lines = self.metrics.code_lines.saturating_add(1);
70            }
71            LineClass::Comment => {
72                self.metrics.comment_lines = self.metrics.comment_lines.saturating_add(1);
73            }
74            LineClass::Blank => {
75                self.metrics.code_blank_lines = self.metrics.code_blank_lines.saturating_add(1);
76            }
77        }
78        self.line.clear();
79    }
80}
81
82#[derive(Clone, Copy, Debug)]
83#[allow(clippy::struct_excessive_bools)]
84struct Syntax {
85    line_comments: &'static [&'static [u8]],
86    block: Option<BlockSyntax>,
87    nested_blocks: bool,
88    triple_quotes: bool,
89    backtick_strings: bool,
90    rust_raw_strings: bool,
91    shell_hash_boundary: bool,
92    ruby_blocks: bool,
93}
94
95impl Syntax {
96    fn for_type(file_type: &str) -> Option<Self> {
97        let c_like = Self {
98            line_comments: &[b"//"],
99            block: Some(BlockSyntax { open: b"/*", close: b"*/" }),
100            nested_blocks: false,
101            triple_quotes: false,
102            backtick_strings: false,
103            rust_raw_strings: false,
104            shell_hash_boundary: false,
105            ruby_blocks: false,
106        };
107        match file_type {
108            "rust" => Some(Self { nested_blocks: true, rust_raw_strings: true, ..c_like }),
109            "javascript" | "typescript" | "go" => Some(Self { backtick_strings: true, ..c_like }),
110            "c" | "cpp" | "csharp" | "java" => Some(c_like),
111            "kotlin" | "swift" => Some(Self { nested_blocks: true, triple_quotes: true, ..c_like }),
112            "php" => Some(Self { line_comments: &[b"//", b"#"], ..c_like }),
113            "python" | "ruby" => Some(Self {
114                line_comments: &[b"#"],
115                block: None,
116                nested_blocks: false,
117                triple_quotes: true,
118                backtick_strings: false,
119                rust_raw_strings: false,
120                shell_hash_boundary: false,
121                ruby_blocks: file_type == "ruby",
122            }),
123            "shell" => Some(Self {
124                line_comments: &[b"#"],
125                block: None,
126                nested_blocks: false,
127                triple_quotes: false,
128                backtick_strings: true,
129                rust_raw_strings: false,
130                shell_hash_boundary: true,
131                ruby_blocks: false,
132            }),
133            "sql" => Some(Self {
134                line_comments: &[b"--"],
135                block: Some(BlockSyntax { open: b"/*", close: b"*/" }),
136                nested_blocks: false,
137                triple_quotes: false,
138                backtick_strings: true,
139                rust_raw_strings: false,
140                shell_hash_boundary: false,
141                ruby_blocks: false,
142            }),
143            _ => None,
144        }
145    }
146}
147
148#[derive(Clone, Copy, Debug)]
149struct BlockSyntax {
150    open: &'static [u8],
151    close: &'static [u8],
152}
153
154#[derive(Clone, Copy, Debug)]
155enum State {
156    Normal,
157    BlockComment { depth: u16 },
158    Quoted { quote: u8, escaped: bool, multiline: bool },
159    TripleQuoted { quote: u8 },
160    RustRaw { hashes: u8 },
161    RubyBlock,
162}
163
164#[derive(Clone, Copy, Debug)]
165enum LineClass {
166    Code,
167    Comment,
168    Blank,
169}
170
171fn classify_line(syntax: Syntax, state: &mut State, line: &[u8]) -> LineClass {
172    let mut index = usize::from(line.starts_with(&[0xef, 0xbb, 0xbf]));
173    index = index.saturating_mul(3);
174    let mut whitespace_boundary = matches!(state, State::Normal);
175    let mut code =
176        matches!(state, State::Quoted { .. } | State::TripleQuoted { .. } | State::RustRaw { .. });
177    let mut comment = matches!(state, State::BlockComment { .. });
178
179    if matches!(state, State::RubyBlock) {
180        if line.starts_with(b"=end") {
181            *state = State::Normal;
182        }
183        return LineClass::Comment;
184    }
185    if syntax.ruby_blocks && matches!(state, State::Normal) && line.starts_with(b"=begin") {
186        *state = State::RubyBlock;
187        return LineClass::Comment;
188    }
189
190    while index < line.len() {
191        match *state {
192            State::BlockComment { mut depth } => {
193                comment = true;
194                let block = syntax.block.expect("block state requires block syntax");
195                if syntax.nested_blocks && line[index..].starts_with(block.open) {
196                    depth = depth.saturating_add(1);
197                    *state = State::BlockComment { depth };
198                    index += block.open.len();
199                } else if line[index..].starts_with(block.close) {
200                    depth = depth.saturating_sub(1);
201                    *state = if depth == 0 { State::Normal } else { State::BlockComment { depth } };
202                    index += block.close.len();
203                } else {
204                    index += 1;
205                }
206            }
207            State::Quoted { quote, mut escaped, multiline } => {
208                code = true;
209                let byte = line[index];
210                if escaped {
211                    escaped = false;
212                } else if byte == b'\\' {
213                    escaped = true;
214                } else if byte == quote {
215                    *state = State::Normal;
216                    index += 1;
217                    continue;
218                }
219                *state = State::Quoted { quote, escaped, multiline };
220                index += 1;
221            }
222            State::TripleQuoted { quote } => {
223                code = true;
224                if line[index..].starts_with(&[quote, quote, quote]) {
225                    *state = State::Normal;
226                    index += 3;
227                } else {
228                    index += 1;
229                }
230            }
231            State::RustRaw { hashes } => {
232                code = true;
233                if rust_raw_close(&line[index..], hashes) {
234                    *state = State::Normal;
235                    index += usize::from(hashes) + 1;
236                } else {
237                    index += 1;
238                }
239            }
240            State::RubyBlock => unreachable!("Ruby blocks return before byte scanning"),
241            State::Normal => {
242                let byte = line[index];
243                if byte.is_ascii_whitespace() {
244                    whitespace_boundary = true;
245                    index += 1;
246                    continue;
247                }
248                if let Some((character, width)) = leading_utf8_character(&line[index..]) {
249                    if super::content_basic_metrics::is_content_whitespace(character) {
250                        whitespace_boundary = true;
251                        index += width;
252                        continue;
253                    }
254                }
255                if syntax.line_comments.iter().any(|marker| {
256                    line[index..].starts_with(marker)
257                        && (!syntax.shell_hash_boundary || whitespace_boundary)
258                }) {
259                    comment = true;
260                    break;
261                }
262                if let Some(block) = syntax.block {
263                    if line[index..].starts_with(block.open) {
264                        comment = true;
265                        whitespace_boundary = false;
266                        *state = State::BlockComment { depth: 1 };
267                        index += block.open.len();
268                        continue;
269                    }
270                }
271                if syntax.rust_raw_strings {
272                    if let Some((hashes, consumed)) = rust_raw_open(&line[index..]) {
273                        code = true;
274                        whitespace_boundary = false;
275                        *state = State::RustRaw { hashes };
276                        index += consumed;
277                        continue;
278                    }
279                }
280                if syntax.triple_quotes
281                    && matches!(byte, b'\'' | b'"')
282                    && line[index..].starts_with(&[byte, byte, byte])
283                {
284                    code = true;
285                    whitespace_boundary = false;
286                    *state = State::TripleQuoted { quote: byte };
287                    index += 3;
288                    continue;
289                }
290                if matches!(byte, b'\'' | b'"') || (syntax.backtick_strings && byte == b'`') {
291                    code = true;
292                    whitespace_boundary = false;
293                    *state = State::Quoted { quote: byte, escaped: false, multiline: byte == b'`' };
294                    index += 1;
295                    continue;
296                }
297                code = true;
298                whitespace_boundary = false;
299                index += 1;
300            }
301        }
302    }
303
304    if let State::Quoted { multiline: false, .. } = state {
305        *state = State::Normal;
306    }
307    if code {
308        LineClass::Code
309    } else if comment {
310        LineClass::Comment
311    } else {
312        LineClass::Blank
313    }
314}
315
316fn leading_utf8_character(input: &[u8]) -> Option<(char, usize)> {
317    let width = match *input.first()? {
318        0x00..=0x7f => 1,
319        0xc2..=0xdf => 2,
320        0xe0..=0xef => 3,
321        0xf0..=0xf4 => 4,
322        _ => return None,
323    };
324    let character = std::str::from_utf8(input.get(..width)?).ok()?.chars().next()?;
325    Some((character, width))
326}
327
328fn rust_raw_open(input: &[u8]) -> Option<(u8, usize)> {
329    if input.first() != Some(&b'r') {
330        return None;
331    }
332    let hashes = input[1..].iter().take_while(|byte| **byte == b'#').count();
333    if hashes > usize::from(u8::MAX) || input.get(hashes + 1) != Some(&b'"') {
334        return None;
335    }
336    Some((u8::try_from(hashes).expect("bounded above"), hashes + 2))
337}
338
339fn rust_raw_close(input: &[u8], hashes: u8) -> bool {
340    input.first() == Some(&b'"')
341        && (hashes == 0
342            || input
343                .get(1..=usize::from(hashes))
344                .is_some_and(|tail| tail.iter().all(|byte| *byte == b'#')))
345}
346
347#[cfg(test)]
348mod tests {
349    use super::*;
350
351    fn count(language: &str, chunks: &[&[u8]]) -> MetricValues {
352        let mut counter = CodeAccumulator::for_type(language).expect("supported language");
353        for chunk in chunks {
354            counter.push(chunk);
355        }
356        counter.finish()
357    }
358
359    #[test]
360    fn partitions_c_like_source_and_counts_mixed_lines_as_code() {
361        let metrics = count(
362            "javascript",
363            &[b"// first\r\nlet url = \"https://example.test\"; // tail\r/* block\n\nend */\n`// text\nmore`;"],
364        );
365        assert_eq!(metrics.physical_lines, 7);
366        assert_eq!(metrics.code_lines, 3);
367        assert_eq!(metrics.comment_lines, 4);
368        assert_eq!(metrics.code_blank_lines, 0);
369    }
370
371    #[test]
372    fn rust_nested_comments_and_raw_strings_ignore_comment_markers() {
373        let source =
374            b"/* outer\n/* inner */\n*/\nlet raw = r##\"/* text */\n// still text\"##;\n\n";
375        let expected = count("rust", &[source]);
376        assert_eq!(expected.physical_lines, 6);
377        assert_eq!(expected.code_lines, 2);
378        assert_eq!(expected.comment_lines, 3);
379        assert_eq!(expected.code_blank_lines, 1);
380        for split in 0..=source.len() {
381            assert_eq!(count("rust", &[&source[..split], &source[split..]]), expected);
382        }
383    }
384
385    #[test]
386    fn triple_quoted_docstrings_are_code_in_v1() {
387        let metrics = count("python", &[b"\"\"\"docs\n# text\n\"\"\"\n# comment\npass\n"]);
388        assert_eq!(metrics.physical_lines, 5);
389        assert_eq!(metrics.code_lines, 4);
390        assert_eq!(metrics.comment_lines, 1);
391        assert_eq!(metrics.code_blank_lines, 0);
392    }
393
394    #[test]
395    fn every_line_ending_convention_has_the_same_partition() {
396        for source in [
397            "// comment\nlet value = 1;\n\n",
398            "// comment\r\nlet value = 1;\r\n\r\n",
399            "// comment\rlet value = 1;\r\r",
400            "// comment\r\nlet value = 1;\r\n",
401        ] {
402            let metrics = count("rust", &[source.as_bytes()]);
403            let expected_lines = if source.ends_with("value = 1;\r\n") { 2 } else { 3 };
404            assert_eq!(metrics.physical_lines, expected_lines, "{source:?}");
405            assert_eq!(metrics.code_lines, 1, "{source:?}");
406            assert_eq!(metrics.comment_lines, 1, "{source:?}");
407            assert_eq!(metrics.code_blank_lines, expected_lines - 2, "{source:?}");
408        }
409    }
410
411    #[test]
412    fn a_leading_utf8_bom_is_not_an_invented_line() {
413        let empty = count("rust", &[b"\xef\xbb\xbf"]);
414        assert_eq!(empty.physical_lines, 0);
415
416        let blank = count("rust", &[b"\xef\xbb\xbf\n"]);
417        assert_eq!(blank.physical_lines, 1);
418        assert_eq!(blank.code_blank_lines, 1);
419    }
420
421    #[test]
422    fn unicode_whitespace_uses_the_basic_analyzers_pinned_table() {
423        let metrics = count("rust", &["\u{3000}\n\u{2003}// comment\n".as_bytes()]);
424        assert_eq!(metrics.physical_lines, 2);
425        assert_eq!(metrics.code_blank_lines, 1);
426        assert_eq!(metrics.comment_lines, 1);
427        assert_eq!(metrics.code_lines, 0);
428
429        let shell = count("shell", &["\u{3000}# comment\n".as_bytes()]);
430        assert_eq!(shell.comment_lines, 1);
431        assert_eq!(shell.code_lines, 0);
432    }
433
434    #[test]
435    fn unsupported_languages_are_explicit() {
436        assert!(CodeAccumulator::for_type("haskell").is_none());
437    }
438}
fdu_core/content/content_code_metrics.rs

fdu_core/content/
content_code_metrics.rs
