Skip to main content

big_code_analysis_ast/
tools.rs

1// Per-language metric and AST modules deliberately consume the macro-
2// generated tree-sitter token enums via `use crate::*` and `use Foo::*`
3// inside match expressions — explicit imports would list dozens of
4// variants per arm and obscure the per-language token sets that are the
5// point of these files. Allowed at the module level rather than per
6// function so the per-language impl blocks stay readable.
7#![allow(clippy::wildcard_imports, clippy::enum_glob_use)]
8// Metric counts (token, function, branch, argument, etc.) are stored as
9// `usize` and crossed with `f64` averages, ratios, and Halstead scores
10// across the cyclomatic / MI / Halstead computations. The `usize as f64`
11// and `f64 as usize` casts are intentional and snapshot-anchored — every
12// site is bounded by the count it came from. Allowing the lints at the
13// module level keeps the metric arithmetic legible.
14#![allow(
15    clippy::cast_precision_loss,
16    clippy::cast_possible_truncation,
17    clippy::cast_sign_loss
18)]
19
20//! Language detection and the file readers.
21
22use std::borrow::Cow;
23use std::cmp::Ordering;
24use std::collections::HashMap;
25use std::fs::{self, File};
26use std::io::{Read, Write};
27use std::path::{Component, Path, PathBuf};
28use std::sync::OnceLock;
29
30use regex::bytes::Regex;
31
32use crate::langs::*;
33
34/// Reads a file, normalising all CR-only and CRLF line endings to LF.
35///
36/// **Note for downstream consumers**: the returned buffer never contains `\r`
37/// bytes. Callers that previously observed raw `\r\n` sequences will see plain
38/// `\n` after this call. This is intentional — the metric engine requires LF-
39/// only input — but it is a behavioural difference from a plain `fs::read`.
40///
41/// # Errors
42///
43/// Returns any [`std::io::Error`] surfaced by [`File::open`] (the
44/// path is missing, lacks read permission, is a directory, …) or by
45/// [`File::read_to_end`] while reading the file contents.
46///
47/// # Examples
48///
49/// ```
50/// use std::path::Path;
51///
52/// use big_code_analysis_ast::read_file;
53///
54/// let path = Path::new("Cargo.toml");
55/// read_file(&path).unwrap();
56/// ```
57pub fn read_file(path: &Path) -> std::io::Result<Vec<u8>> {
58    let mut file = File::open(path)?;
59    let mut data = Vec::new();
60    file.read_to_end(&mut data)?;
61
62    normalize_line_endings(&mut data);
63
64    Ok(data)
65}
66
67/// Bytes from the start of the file probed to decide whether the contents
68/// look like UTF-8 before the whole file is read. A small fixed window keeps
69/// the rejection of obviously-binary files cheap; the last character of the
70/// window may be a multibyte sequence split by this boundary, which the
71/// classifier tolerates only when more file follows (see `read_file_with_eol`).
72const UTF8_PROBE_BYTES: usize = 64;
73
74/// Decides whether a file's probe prefix is decodable UTF-8 and where its
75/// real content starts. Returns the post-BOM content slice when the probe
76/// is acceptable, or the [`SkipReason`] that rejects the file.
77///
78/// A UTF-16 BE/LE BOM marks a file whose body is interleaved-NUL UTF-16,
79/// which the metric engine cannot parse: stripping the BOM and continuing
80/// would let the ASCII-dominant body pass the UTF-8 probe (each NUL is a
81/// valid single-byte UTF-8 scalar) and reach the parser as garbage (issue
82/// #803). Skip such files, mirroring `is_generated`'s documented stance
83/// that UTF-16 source is unsupported. A UTF-8 BOM, by contrast, prefixes
84/// genuine UTF-8 and is stripped so the body parses normally.
85/// `starts_with` is bounds-safe for a probe shorter than the BOM.
86///
87/// Validation is at the byte level rather than via a lossy string
88/// round-trip. The probe is only the first `UTF8_PROBE_BYTES`, so a file
89/// longer than the probe may legitimately have its last multibyte
90/// character split across the window boundary. `String::from_utf8_lossy`
91/// could not distinguish that benign truncation (issue #746) from a real
92/// encoding error, and its replacement character `U+FFFD` collided with
93/// the same scalar appearing legitimately in the source (issue #758).
94/// `probe_truncated` is true only when the file continues past the probe;
95/// when the probe is the whole file there is no more data to complete a
96/// trailing partial sequence, so such a sequence is genuine corruption.
97fn probe_decodable_prefix(
98    start: &[u8],
99    file_size: usize,
100    probe_len: usize,
101) -> Result<&[u8], SkipReason> {
102    let start = if start.starts_with(b"\xFE\xFF") || start.starts_with(b"\xFF\xFE") {
103        return Err(SkipReason::Utf16Bom);
104    } else if let Some(rest) = start.strip_prefix(b"\xEF\xBB\xBF") {
105        rest
106    } else {
107        start
108    };
109
110    let probe_truncated = file_size > probe_len;
111    match std::str::from_utf8(start) {
112        Ok(_) => {}
113        // Only a trailing incomplete multibyte sequence: the bytes before
114        // `valid_up_to()` are valid UTF-8 and the truncated tail is
115        // completed by data later in the file.
116        Err(e) if e.error_len().is_none() && probe_truncated => {}
117        Err(_) => return Err(SkipReason::NotUtf8),
118    }
119    Ok(start)
120}
121
122/// Why [`read_file_with_eol_classified`] declined to hand a file's bytes to
123/// the metric engine.
124///
125/// [`read_file_with_eol`] collapses every one of these into `Ok(None)`, which
126/// left front-ends unable to name the real cause — `bca` reported a
127/// multi-kilobyte binary and a one-byte source alike as "empty" (#1287). This
128/// enum is the classified form of that same decision; the variants map 1:1 onto
129/// the skip branches of [`read_file_with_eol_classified`].
130///
131/// Marked `#[non_exhaustive]`: a future gate (a size ceiling, a further
132/// encoding) adds a variant additively, so match on it with a wildcard arm or
133/// render it through [`Display`](std::fmt::Display).
134#[non_exhaustive]
135#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
136pub enum SkipReason {
137    /// The file is zero bytes long.
138    Empty,
139    /// The file holds 1–3 bytes — too little to be worth parsing — or it
140    /// shrank below the UTF-8 probe window between the `stat` and the read.
141    TooSmall,
142    /// The file opens with a UTF-16 BE or LE byte-order mark. Its
143    /// interleaved-NUL body would pass a byte-level UTF-8 check and reach the
144    /// parser as garbage, so it is skipped rather than decoded (#803).
145    Utf16Bom,
146    /// The file's leading window is not valid UTF-8 — the shape a binary file
147    /// takes here.
148    NotUtf8,
149}
150
151impl SkipReason {
152    /// Which small-file reason applies to a file of `size` bytes: zero bytes
153    /// is [`Empty`](Self::Empty), anything else under the gate is
154    /// [`TooSmall`](Self::TooSmall).
155    fn for_size(size: usize) -> Self {
156        if size == 0 {
157            Self::Empty
158        } else {
159            Self::TooSmall
160        }
161    }
162}
163
164impl std::fmt::Display for SkipReason {
165    /// Renders the reason as a noun phrase naming the skipped file, so a
166    /// caller can splice it into a diagnostic: `skipping {reason}: {path}`.
167    ///
168    /// Per the project diagnostic convention the phrase carries no severity
169    /// word — the presenting layer owns that prefix.
170    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
171        let phrase = match self {
172            Self::Empty => "empty file",
173            Self::TooSmall => "file too small to analyze (3 bytes or fewer)",
174            Self::Utf16Bom => "UTF-16 file (unsupported encoding)",
175            Self::NotUtf8 => "file with non-UTF-8 contents",
176        };
177        f.write_str(phrase)
178    }
179}
180
181/// Reads a file, normalising all CR-only and CRLF line endings to LF, and ensures
182/// the buffer ends with exactly one `\n`. Returns `None` for readable files
183/// ≤ 3 bytes or files that appear to be non-UTF-8.
184///
185/// `None` names no reason. Use [`read_file_with_eol_classified`] — of which
186/// this function is a thin projection — when the caller reports the skip to a
187/// user, so an empty file, a one-byte source, a binary blob and a UTF-16 file
188/// are not all announced as "empty" (#1287).
189///
190/// # Errors
191///
192/// Returns any [`std::io::Error`] surfaced by [`File::open`] (the
193/// path is missing, lacks read permission, …) or by the subsequent
194/// reads from the open file handle. A directory is always an error, on
195/// every filesystem: Windows refuses the open, Unix the first read
196/// (`IsADirectory`). A clean short read during the probe
197/// (`UnexpectedEof`) yields `Ok(None)`; any other `read_exact` error
198/// kind propagates as `Err`. A non-UTF-8 head, a too-small file, or a
199/// UTF-16 BE/LE BOM is reported via `Ok(None)`, not an error — but
200/// a tiny regular file is probed for readability first, so an
201/// unreadable one errors instead (#1060), and a small-`st_size`
202/// directory is probed the same way so it errors too (#1762). A tiny
203/// FIFO, socket or device is still skipped unopened.
204///
205/// # Examples
206///
207/// ```
208/// use std::path::Path;
209///
210/// use big_code_analysis_ast::read_file_with_eol;
211///
212/// let path = Path::new("Cargo.toml");
213/// read_file_with_eol(&path).unwrap();
214/// ```
215pub fn read_file_with_eol(path: &Path) -> std::io::Result<Option<Vec<u8>>> {
216    read_file_with_eol_classified(path).map(Result::ok)
217}
218
219/// Reads a file exactly as [`read_file_with_eol`] does, but names the reason
220/// when the file is skipped instead of collapsing every skip into `None`.
221///
222/// The outer `Result` is the I/O outcome; the inner one is the analysis
223/// outcome — `Ok(bytes)` for a file the metric engine can parse, or
224/// `Err(`[`SkipReason`]`)` for one it declines. [`read_file_with_eol`] is
225/// `read_file_with_eol_classified(path).map(Result::ok)`, so the two agree
226/// branch for branch.
227///
228/// # Errors
229///
230/// Identical to [`read_file_with_eol`]: any [`std::io::Error`] from
231/// [`File::open`] or the subsequent reads propagates, and a directory is
232/// always one of those (the open fails on Windows, the first read on Unix).
233/// Skips are *not* errors — they arrive as `Ok(Err(reason))`; a tiny regular
234/// file is probed for readability before it is skipped.
235///
236/// # Examples
237///
238/// ```
239/// use std::path::Path;
240///
241/// use big_code_analysis_ast::{SkipReason, read_file_with_eol_classified};
242///
243/// let dir = tempfile::tempdir().unwrap();
244/// let path = dir.path().join("tiny.rs");
245/// std::fs::write(&path, b"x").unwrap();
246///
247/// assert_eq!(
248///     read_file_with_eol_classified(&path).unwrap(),
249///     Err(SkipReason::TooSmall)
250/// );
251/// ```
252pub fn read_file_with_eol_classified(path: &Path) -> std::io::Result<Result<Vec<u8>, SkipReason>> {
253    match read_gated(path) {
254        Ok(data) => Ok(Ok(data)),
255        Err(ReadStop::Skip(reason)) => Ok(Err(reason)),
256        Err(ReadStop::Io(err)) => Err(err),
257    }
258}
259
260/// The two ways [`read_gated`] stops short of returning bytes. Collapsing
261/// them into one `Err` is what lets that function spell every gate as `?`
262/// instead of hand-wrapping each early return in the public
263/// `Ok(Err(reason))` / `Err(err)` shape; the split back out happens once, in
264/// [`read_file_with_eol_classified`].
265enum ReadStop {
266    /// A genuine I/O failure, propagated to the caller as `Err`.
267    Io(std::io::Error),
268    /// The file was read far enough to decide the metric engine should not
269    /// see it. Not an error (issue #804).
270    Skip(SkipReason),
271}
272
273impl From<std::io::Error> for ReadStop {
274    fn from(err: std::io::Error) -> Self {
275        Self::Io(err)
276    }
277}
278
279impl From<SkipReason> for ReadStop {
280    fn from(reason: SkipReason) -> Self {
281        Self::Skip(reason)
282    }
283}
284
285/// Applies the pre-parse gates in order — size, readability, UTF-8 probe —
286/// and returns the EOL-normalised contents of a file that clears them all.
287fn read_gated(path: &Path) -> Result<Vec<u8>, ReadStop> {
288    let meta = fs::metadata(path);
289    let file_size = meta.as_ref().map_or(1024 * 1024, |m| m.len() as usize);
290    if file_size <= 3 {
291        // Nothing worth parsing this small — but `stat` alone must not
292        // decide it: `stat` succeeds on a file the process cannot open,
293        // so an unreadable tiny file read as empty and `bca check`
294        // exited 0 on a tree it never read (#1060). `meta` is
295        // necessarily `Ok` here; the open-and-read is a discarded
296        // readability probe, skipped for a FIFO, socket or device, where
297        // the open or the read can block or fail spuriously. A
298        // directory takes the probe too: its `st_size` is under the
299        // gate on btrfs, ZFS and NTFS, and only the probe turns that
300        // into the platform's directory error — Windows refuses the
301        // open, Unix the first read (#1762).
302        if meta.is_ok_and(|m| m.is_file() || m.is_dir()) {
303            File::open(path).and_then(|mut file| file.read(&mut [0]))?;
304        }
305        return Err(SkipReason::for_size(file_size).into());
306    }
307
308    let mut file = File::open(path)?;
309
310    let probe_len = UTF8_PROBE_BYTES.min(file_size);
311    let mut start = vec![0; probe_len];
312    // A clean short read (the file shrank below the probe between the
313    // `metadata` call and here) is reported as a skip, matching the
314    // too-small-file case. Any other `read_exact` failure — a real I/O
315    // fault such as a permission or hardware error — must propagate as
316    // `Err` per the documented contract (issue #804); collapsing every
317    // error to a skip would silently swallow genuine read failures.
318    match file.read_exact(&mut start) {
319        Ok(()) => {}
320        Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
321            return Err(SkipReason::TooSmall.into());
322        }
323        Err(e) => return Err(e.into()),
324    }
325
326    // Sniff the probe: reject UTF-16 / corrupt files, strip a UTF-8 BOM,
327    // and anchor the buffer at the post-BOM content. An `Err` names why
328    // the file is skipped (see `probe_decodable_prefix`).
329    let start = probe_decodable_prefix(&start, file_size, probe_len)?;
330
331    let mut data = Vec::with_capacity(file_size + 2);
332    data.extend_from_slice(start);
333
334    file.read_to_end(&mut data)?;
335
336    normalize_line_endings(&mut data);
337
338    Ok(data)
339}
340
341/// Normalises an in-memory source buffer to match the [`read_file_with_eol`]
342/// on-disk path: all CR-only and CRLF line endings become LF, and the buffer is
343/// guaranteed to end with exactly one `\n`.
344///
345/// In-memory entry points (the web server's JSON/octet-stream payloads, the
346/// Python `analyze_source` bindings) feed caller-supplied bytes straight to the
347/// parser, whereas the CLI reads files through [`read_file_with_eol`]. Without
348/// this step, identical content yields different metrics across surfaces — an
349/// editor buffer with no trailing newline reports `sloc: 0` over the wire but
350/// `sloc: 1` from the CLI on the same bytes (issue #640). Run the buffer through
351/// this helper before parsing so every surface computes the canonical numbers.
352///
353/// Returns a fresh owned buffer; the input is consumed so the common case
354/// (already-owned request body) reuses its allocation.
355///
356/// # Examples
357///
358/// ```
359/// use big_code_analysis_ast::normalize_eol;
360///
361/// // CRLF endings collapse to LF and a missing final newline is added.
362/// assert_eq!(normalize_eol(b"a\r\nb".to_vec()), b"a\nb\n");
363/// ```
364#[must_use]
365pub fn normalize_eol(mut data: Vec<u8>) -> Vec<u8> {
366    normalize_line_endings(&mut data);
367    data
368}
369
370/// Writes data to a file.
371///
372/// # Errors
373///
374/// Returns any [`std::io::Error`] surfaced by [`File::create`]
375/// (parent directory missing, lacks write permission, target is a
376/// directory, …) or by [`File::write_all`] while writing the buffer.
377///
378/// # Examples
379///
380/// ```no_run
381/// use std::path::Path;
382///
383/// use big_code_analysis_ast::write_file;
384///
385/// let path = Path::new("foo.txt");
386/// let data: [u8; 4] = [0; 4];
387/// write_file(&path, &data).unwrap();
388/// ```
389pub fn write_file(path: &Path, data: &[u8]) -> std::io::Result<()> {
390    let mut file = File::create(path)?;
391    file.write_all(data)?;
392
393    Ok(())
394}
395
396/// Detects the language of a code using
397/// the extension of a file.
398///
399/// # Examples
400///
401/// ```
402/// use std::path::Path;
403///
404/// use big_code_analysis_ast::get_language_for_file;
405///
406/// let path = Path::new("build.rs");
407/// get_language_for_file(&path).unwrap();
408/// ```
409#[must_use]
410pub fn get_language_for_file(path: &Path) -> Option<LANG> {
411    let ext = path.extension()?.to_str()?;
412    get_from_ext(&lowercase_ext(ext))
413}
414
415/// Borrows `ext` when it is already the ASCII-lowercase spelling
416/// [`get_from_ext`]'s table is keyed on, allocating only for the
417/// mixed-case minority (`foo.C`, `foo.PY`).
418///
419/// The `is_ascii` guard is load-bearing: [`str::to_lowercase`] folds
420/// `U+212A KELVIN SIGN` onto ASCII `k`, so a non-ASCII extension must
421/// keep that Unicode-aware path to leave the lookup key as it was before
422/// #1111. Past the guard the input is ASCII, hence the cheaper fold.
423fn lowercase_ext(ext: &str) -> Cow<'_, str> {
424    if !ext.is_ascii() {
425        Cow::Owned(ext.to_lowercase())
426    } else if ext.bytes().any(|b| b.is_ascii_uppercase()) {
427        Cow::Owned(ext.to_ascii_lowercase())
428    } else {
429        Cow::Borrowed(ext)
430    }
431}
432
433fn mode_to_str(mode: &[u8]) -> Option<String> {
434    std::str::from_utf8(mode).ok().map(str::to_lowercase)
435}
436
437// comment containing coding info are useful
438static RE1_EMACS: OnceLock<Regex> = OnceLock::new();
439static RE2_EMACS: OnceLock<Regex> = OnceLock::new();
440static RE1_VIM: OnceLock<Regex> = OnceLock::new();
441static RE_GENERATED: OnceLock<Regex> = OnceLock::new();
442
443// Regular expressions
444const FIRST_EMACS_EXPRESSION: &str = r"(?i)-\*-.*[^-\w]mode\s*:\s*([^:;\s]+)";
445const SECOND_EMACS_EXPRESSION: &str = r"-\*-\s*([^:;\s]+)\s*-\*-";
446const VIM_EXPRESSION: &str = r"(?i)vim\s*:.*[^\w]ft\s*=\s*([^:\s]+)";
447
448// Generated-code marker patterns. Matched against the leading window of the
449// file (see `is_generated`) so a marker phrase deep in the body does not
450// trigger a skip. Each alternative covers a widely-used convention:
451//
452// - `@generated`      — Facebook / Meta convention, also used by buck2,
453//                       rustfmt, prettier, and many code generators.
454// - `DO NOT EDIT`     — Go's `Code generated ... DO NOT EDIT.` line is
455//                       canonical, but the bare phrase appears in Bazel,
456//                       protoc, OpenAPI clients, etc. — match either.
457// - `GENERATED CODE`  — Lizard's marker; preserved for compatibility with
458//                       projects that already tag generated files this way.
459const GENERATED_EXPRESSION: &str = r"(?i)@generated\b|DO NOT EDIT|GENERATED CODE";
460
461/// Bytes from the start of the file scanned for a generated-code marker.
462/// 5 KiB is enough to cover any reasonable file header (license + autogen
463/// preamble) without paying a meaningful read cost.
464const GENERATED_SCAN_BYTES: usize = 5 * 1024;
465/// Maximum lines scanned for a generated-code marker. Caps the work on a
466/// pathological "all-on-one-line" file.
467const GENERATED_SCAN_LINES: usize = 50;
468
469/// Returns `true` when `buf` looks like generated code: its leading window
470/// (first ~50 lines or first 5 KiB, whichever is smaller) contains a known
471/// marker phrase. Matching is case-insensitive for the marker and never
472/// allocates on the negative path.
473///
474/// Recognized markers:
475///
476/// - `@generated` — Facebook / Meta convention, also used by buck2,
477///   rustfmt, and prettier.
478/// - `DO NOT EDIT` — Go's `Code generated by ... DO NOT EDIT.` is the
479///   canonical form; the bare phrase is also widely copied.
480/// - `GENERATED CODE` — Lizard's marker, preserved for compatibility.
481///
482/// Detection runs against raw bytes before parsing, so callers can discard
483/// generated files without paying tree-sitter parse cost. Non-UTF-8 input
484/// will not panic — `regex::bytes::Regex` operates on the raw byte slice.
485///
486/// # Examples
487///
488/// ```
489/// use big_code_analysis_ast::is_generated;
490///
491/// assert!(is_generated(b"// @generated\nfn x() {}\n"));
492/// assert!(is_generated(
493///     b"// Code generated by protoc. DO NOT EDIT.\npackage x\n",
494/// ));
495/// assert!(!is_generated(b"fn main() { /* not generated */ }\n"));
496/// ```
497///
498/// # Panics
499///
500/// Panics if the embedded marker regex set fails to build; the marker
501/// list is a static literal so this represents a compile-time bug, not
502/// a runtime input that can be handled.
503pub fn is_generated(buf: &[u8]) -> bool {
504    // Strip a leading UTF-8 BOM so a marker on the first line of a
505    // BOM-prefixed file still matches against the line start. UTF-16 BOMs
506    // are not handled: the byte-pattern regex cannot match the
507    // interleaved-zero encoding (`@\x00g\x00...`) that follows a UTF-16
508    // BOM, so a strip would not enable detection — it would only obscure
509    // the fact that UTF-16 source files are unsupported here.
510    let buf = buf.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(buf);
511
512    // Bound the search window: at most GENERATED_SCAN_BYTES bytes, and
513    // among those, stop after GENERATED_SCAN_LINES newlines. Scanning fewer
514    // lines avoids matching a marker phrase deep in the file body (the
515    // negative case in the issue's acceptance criteria).
516    let cap = buf.len().min(GENERATED_SCAN_BYTES);
517    let end = buf[..cap]
518        .iter()
519        .enumerate()
520        .filter_map(|(i, &b)| (b == b'\n').then_some(i + 1))
521        .nth(GENERATED_SCAN_LINES - 1)
522        .unwrap_or(cap);
523    let window = &buf[..end];
524
525    RE_GENERATED
526        .get_or_init(|| {
527            Regex::new(GENERATED_EXPRESSION).expect("GENERATED_EXPRESSION is a constant regex")
528        })
529        .is_match(window)
530}
531
532#[inline]
533fn get_regex<'a>(
534    once_lock: &OnceLock<Regex>,
535    line: &'a [u8],
536    regex: &'a str,
537) -> Option<regex::bytes::Captures<'a>> {
538    once_lock
539        .get_or_init(|| Regex::new(regex).expect("constant regex pattern must compile"))
540        .captures(line)
541}
542
543/// Resolves a language from a script's shebang line.
544///
545/// Returns `None` unless `buf` starts with `#!`. Reads up to the first `\n`,
546/// strips an optional trailing `\r`, splits on whitespace, and takes the
547/// basename of either the first token or — when that basename is `env` — the
548/// next non-flag token. Trailing version digits and dots (`python3`,
549/// `lua5.1`, `perl5.36`) are stripped before lookup. Non-UTF-8 bytes on the
550/// shebang line yield `None` (no panic).
551fn get_shebang_lang(buf: &[u8]) -> Option<LANG> {
552    // Early-out for the common case (any non-shebang buffer): no allocation,
553    // no UTF-8 decoding.
554    let rest = buf.strip_prefix(b"#!")?;
555    let line_end = rest.iter().position(|&b| b == b'\n').unwrap_or(rest.len());
556    let line = &rest[..line_end];
557    // Trim a trailing CR even though normalize_line_endings should have removed
558    // it — guess_language is on the public API and may be called with raw input.
559    let line = line.strip_suffix(b"\r").unwrap_or(line);
560    let line = std::str::from_utf8(line).ok()?;
561
562    let mut tokens = line.split_ascii_whitespace();
563    let first_base = basename(tokens.next()?);
564
565    let interpreter = if first_base == "env" {
566        skip_env_args(&mut tokens)?
567    } else {
568        first_base
569    };
570
571    get_from_interpreter(strip_version_suffix(interpreter))
572}
573
574// Walk past leading `env` arguments (`-FLAG`, `-u VAR`, `NAME=value`) and
575// return the basename of the actual interpreter token. Per `env(1)`, only
576// `-u` consumes a following argument; other short flags (`-i`, `-S`, …)
577// stand alone or carry their argument inline (e.g. `-S "node --foo"`).
578fn skip_env_args<'a>(tokens: &mut std::str::SplitAsciiWhitespace<'a>) -> Option<&'a str> {
579    loop {
580        let tok = tokens.next()?;
581        if let Some(flag) = tok.strip_prefix('-') {
582            if flag == "u" {
583                tokens.next()?;
584            }
585            continue;
586        }
587        if tok.contains('=') {
588            continue;
589        }
590        return Some(basename(tok));
591    }
592}
593
594fn basename(path: &str) -> &str {
595    path.rsplit_once('/').map_or(path, |(_, name)| name)
596}
597
598/// Strips a trailing run of digits and dots used to encode an interpreter
599/// version (`python3` → `python`, `lua5.1` → `lua`, `perl5.36` → `perl`).
600fn strip_version_suffix(name: &str) -> &str {
601    let trimmed = name.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.');
602    if trimmed.is_empty() { name } else { trimmed }
603}
604
605fn get_from_interpreter(name: &str) -> Option<LANG> {
606    match name {
607        "sh" | "bash" | "dash" | "ksh" | "zsh" => Some(LANG::Bash),
608        "python" => Some(LANG::Python),
609        "perl" => Some(LANG::Perl),
610        "lua" | "luajit" => Some(LANG::Lua),
611        "php" | "php-cgi" => Some(LANG::Php),
612        "node" | "nodejs" => Some(LANG::Javascript),
613        "tclsh" | "wish" => Some(LANG::Tcl),
614        "ruby" => Some(LANG::Ruby),
615        "elixir" | "iex" => Some(LANG::Elixir),
616        _ => None,
617    }
618}
619
620// Editors place mode/file-local-variable lines near the very top or
621// very bottom of a file. Emacs honours the first non-shebang line and a
622// trailing "Local Variables:" block; Vim honours modelines in the first
623// or last few lines (`modelines` defaults to 5). Scanning this many real
624// lines at each end covers both conventions without trawling the body.
625const MODE_LINE_SCAN_WINDOW: usize = 5;
626
627// Entries into `get_emacs_mode` — the only thing able to observe #1111's
628// laziness: every assertion on what `guess_language` returns holds just as
629// well when the scan runs eagerly and throws the result away. Read by the
630// test that pins it, which carries why.
631crate::observation::counter!(modeline_scans);
632
633fn get_emacs_mode(buf: &[u8]) -> Option<String> {
634    modeline_scans::record();
635    // Forward scan: the first `MODE_LINE_SCAN_WINDOW` real lines may carry
636    // an emacs `-*- … -*-` header or a Vim modeline. `split` yields one
637    // element per line (no unbounded remainder), and `take` bounds the
638    // window precisely — the former `splitn(5)` + `i == 3` break inspected
639    // only 4 lines yet split off a 5th unbounded remainder (issue #709).
640    for line in buf.split(|c| *c == b'\n').take(MODE_LINE_SCAN_WINDOW) {
641        if let Some(cap) = get_regex(&RE1_EMACS, line, FIRST_EMACS_EXPRESSION) {
642            return mode_to_str(&cap[1]);
643        } else if let Some(cap) = get_regex(&RE2_EMACS, line, SECOND_EMACS_EXPRESSION) {
644            return mode_to_str(&cap[1]);
645        } else if let Some(cap) = get_regex(&RE1_VIM, line, VIM_EXPRESSION) {
646            return mode_to_str(&cap[1]);
647        }
648    }
649
650    // Backward scan for a trailing Vim modeline. Skip empty pieces so a
651    // trailing newline (the common case after `read_file_with_eol`) and
652    // any trailing blank lines do not consume the window before a real
653    // modeline is reached — the former `rsplitn(5)` spent its first slot
654    // on that empty piece, covering fewer than the intended real lines.
655    for line in buf
656        .rsplit(|c| *c == b'\n')
657        .filter(|line| !line.is_empty())
658        .take(MODE_LINE_SCAN_WINDOW)
659    {
660        if let Some(cap) = get_regex(&RE1_VIM, line, VIM_EXPRESSION) {
661            return mode_to_str(&cap[1]);
662        }
663    }
664
665    None
666}
667
668/// Guesses the language of a code.
669///
670/// Returns a tuple containing a [`LANG`] as first argument
671/// and the language name as a second one.
672///
673/// # Examples
674///
675/// ```
676/// use std::path::PathBuf;
677///
678/// use big_code_analysis_ast::guess_language;
679///
680/// let source_code = "int a = 42;";
681///
682/// // The path to a dummy file used to contain the source code
683/// let path = PathBuf::from("foo.c");
684/// let source_slice = source_code.as_bytes();
685///
686/// // Guess the language of a code
687/// guess_language(&source_slice, &path);
688/// ```
689///
690/// [`LANG`]: enum.LANG.html
691pub fn guess_language<P: AsRef<Path>>(buf: &[u8], path: P) -> (Option<LANG>, &'static str) {
692    // Precedence: extension, then emacs/vim modeline, then shebang. The
693    // fallbacks are lazy, so a recognised extension never pays for the
694    // modeline scan; the previous form ran it for every file and
695    // discarded it, every arm of its extension branch having returned the
696    // extension's language — the "modeline agrees" arm included (#1111).
697    let lang = get_language_for_file(path.as_ref())
698        .or_else(|| get_emacs_mode(buf).and_then(|mode| get_from_emacs_mode(&mode)))
699        .or_else(|| get_shebang_lang(buf));
700
701    lang.map_or((None, ""), |lang| (Some(lang), lang.name()))
702}
703
704/// Normalises all CR-only and CRLF line endings to LF throughout the buffer,
705/// then ensures the buffer ends with exactly one `\n`.
706pub fn normalize_line_endings(data: &mut Vec<u8>) {
707    // In-place compaction: write pointer stays ≤ read pointer, so no extra allocation.
708    let mut w = 0;
709    let mut r = 0;
710    while r < data.len() {
711        if data[r] == b'\r' {
712            data[w] = b'\n';
713            w += 1;
714            r += if data.get(r + 1).copied() == Some(b'\n') {
715                2
716            } else {
717                1
718            };
719        } else {
720            data[w] = data[r];
721            w += 1;
722            r += 1;
723        }
724    }
725    data.truncate(w);
726    let trailing = data.iter().rev().take_while(|&&c| c == b'\n').count();
727    data.truncate(data.len() - trailing);
728    data.push(b'\n');
729}
730
731pub(crate) fn normalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
732    // Copied from Cargo sources: https://github.com/rust-lang/cargo/blob/master/src/cargo/util/paths.rs#L65
733    let mut components = path.as_ref().components().peekable();
734    let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().copied() {
735        components.next();
736        PathBuf::from(c.as_os_str())
737    } else {
738        PathBuf::new()
739    };
740
741    for component in components {
742        match component {
743            // A `Prefix` (Windows drive / UNC) component only ever
744            // appears first; the leading peek+next above already
745            // consumed it, so it cannot recur in this loop.
746            Component::Prefix(..) => unreachable!(),
747            Component::RootDir => {
748                ret.push(component.as_os_str());
749            }
750            Component::CurDir => {}
751            Component::ParentDir => {
752                ret.pop();
753            }
754            Component::Normal(c) => {
755                ret.push(c);
756            }
757        }
758    }
759    ret
760}
761
762pub(crate) fn get_paths_dist(path1: &Path, path2: &Path) -> Option<usize> {
763    for ancestor in path1.ancestors() {
764        if path2.starts_with(ancestor) && !ancestor.as_os_str().is_empty() {
765            // `ancestor` is yielded by `path1.ancestors()`, so it is
766            // a prefix of `path1` by construction; `path2` was just
767            // verified by `starts_with` above. Both `strip_prefix`
768            // calls are therefore infallible.
769            let path1 = path1
770                .strip_prefix(ancestor)
771                .expect("ancestor is by construction a prefix of path1");
772            let path2 = path2
773                .strip_prefix(ancestor)
774                .expect("ancestor verified by starts_with above");
775            return Some(path1.components().count() + path2.components().count());
776        }
777    }
778    None
779}
780
781pub(crate) fn guess_file<S: ::std::hash::BuildHasher>(
782    current_path: &Path,
783    include_path: &str,
784    all_files: &HashMap<String, Vec<PathBuf>, S>,
785) -> Vec<PathBuf> {
786    let include_path = include_path
787        .strip_prefix("mozilla/")
788        .unwrap_or(include_path);
789
790    // Resolve the include relative to the including file's parent
791    // before normalizing. This preserves leading `..` traversal so
792    // `#include "../foo.h"` from `src/lib/file.c` targets
793    // `src/foo.h`, not the lexically-popped `foo.h` (issue #297).
794    // Lexical-only normalization is required because `current_path`
795    // and the entries in `all_files` are typically not canonicalized
796    // and the included header need not exist on disk yet.
797    let resolved_path = current_path
798        .parent()
799        .map(|parent| normalize_path(parent.join(include_path)));
800
801    let include_path = normalize_path(include_path);
802    let Some(file_name) = include_path.file_name().and_then(|n| n.to_str()) else {
803        return vec![];
804    };
805    let Some(possibilities) = all_files.get(file_name) else {
806        return vec![];
807    };
808    if possibilities.len() == 1 {
809        return possibilities.clone();
810    }
811
812    // Strategy chain: each step looks for a UNIQUE candidate that
813    // matches a progressively weaker signal (full resolved target →
814    // suffix on the normalized include → siblings of the including
815    // file). When no step yields a unique match, fall back to the
816    // closest by path distance, which may return zero or many.
817    resolve_against_resolved(possibilities, current_path, resolved_path.as_deref())
818        .or_else(|| unique_filter(possibilities, current_path, |p| p.ends_with(&include_path)))
819        .or_else(|| resolve_against_parent(possibilities, current_path))
820        .unwrap_or_else(|| min_distance_candidates(possibilities, current_path))
821}
822
823/// Filter `possibilities` to those satisfying `pred` and distinct
824/// from `current_path`, returning `Some(matched)` only when exactly
825/// one survives. The cascading caller treats `None` as "this strategy
826/// did not yield a unique resolution — try the next one."
827fn unique_filter<F>(possibilities: &[PathBuf], current_path: &Path, pred: F) -> Option<Vec<PathBuf>>
828where
829    F: Fn(&PathBuf) -> bool,
830{
831    let matched: Vec<PathBuf> = possibilities
832        .iter()
833        .filter(|p| current_path != p.as_path() && pred(p))
834        .cloned()
835        .collect();
836    (matched.len() == 1).then_some(matched)
837}
838
839/// Strongest signal: a candidate matches the fully resolved relative
840/// target. Prefer exact equality, then suffix match (so absolute
841/// `all_files` entries still match a relative resolved target like
842/// `src/foo.h`).
843fn resolve_against_resolved(
844    possibilities: &[PathBuf],
845    current_path: &Path,
846    resolved: Option<&Path>,
847) -> Option<Vec<PathBuf>> {
848    let resolved = resolved?;
849    unique_filter(possibilities, current_path, |p| p == resolved)
850        .or_else(|| unique_filter(possibilities, current_path, |p| p.ends_with(resolved)))
851}
852
853/// Candidate-in-same-directory heuristic: keep entries whose path
854/// starts with the including file's parent directory.
855fn resolve_against_parent(possibilities: &[PathBuf], current_path: &Path) -> Option<Vec<PathBuf>> {
856    let parent = current_path.parent()?;
857    unique_filter(possibilities, current_path, |p| p.starts_with(parent))
858}
859
860/// Last-chance fallback in the `guess_file` strategy chain: returns
861/// every candidate whose `get_paths_dist` from `current_path` ties
862/// the minimum, or an empty `Vec` when no candidate has a defined
863/// distance. Unlike the unique-match strategies, this may
864/// legitimately return zero or many entries — its result is the
865/// function's final answer, not a "try the next strategy" signal.
866fn min_distance_candidates(possibilities: &[PathBuf], current_path: &Path) -> Vec<PathBuf> {
867    // Hold survivors as borrows during the walk: `Less` arms clear the
868    // prior set without dropping owned `PathBuf`s, and the trailing
869    // `cloned()` runs exactly once per final survivor — never on
870    // entries that were tentatively kept and later evicted.
871    let mut dist_min = usize::MAX;
872    let mut path_min: Vec<&PathBuf> = Vec::new();
873    for p in possibilities {
874        if current_path == p {
875            continue;
876        }
877        let Some(dist) = get_paths_dist(current_path, p) else {
878            continue;
879        };
880        match dist.cmp(&dist_min) {
881            Ordering::Less => {
882                dist_min = dist;
883                path_min.clear();
884                path_min.push(p);
885            }
886            Ordering::Equal => path_min.push(p),
887            Ordering::Greater => {}
888        }
889    }
890    path_min.into_iter().cloned().collect()
891}
892
893#[cfg(test)]
894#[path = "tools_tests.rs"]
895mod tests;