mezura-core 1.1.1

// Reading one file and deciding what each of its lines is: code, comment, or neither.
//
// A line is scanned once for every symbol the language declares, in as few memchr passes as the
// symbols allow, and what comes back are the positions of string delimiters, comment openers and the
// two multiline markers. The rest of the file decides what those positions mean when they overlap,
// which is where every language-specific trap lives.
use std::collections::HashMap;
use std::fs::File;
use std::io::Read as IoRead;
use std::iter::Peekable;
use std::path::Path;
use std::str;
use std::sync::{Arc, LazyLock};
use std::time::Instant;

use memchr::memmem;

use crate::{EngineConfig, Language, LineClass, NestedLanguage, ScanSkip, Span, SpanKind, phase_timing};
use crate::domain::{CommentPair, FileStats, LineContinuation};

pub(crate) const MAX_RETAINED_FILE_BUFFER_BYTES: usize = 4_194_304;

// Real source reaches 350 a line, generated bindings padded into columns, and anything lower
// drops it out of the count. A bundle is in the thousands.
const MINIFIED_AVERAGE_LINE_BYTES : usize = 1_000;
const SMALLEST_FILE_WORTH_TESTING : usize = 10_240;

// Measured over a whole drive: 90% of the markers sit in the first 256 bytes and 92% in the first
// 512. Past that the hits are the generators themselves, whose source holds the marker it prints.
const GENERATED_MARKER_BYTES : usize = 512;
// 'generated by' is deliberately absent: it also matches ordinary sentences, and the Go form
// carries 'DO NOT EDIT' anyway.
const GENERATED_MARKERS : [&str; 4] = ["do not edit", "auto-generated", "autogenerated", "@generated"];
static GENERATED_FINDERS : LazyLock<[memmem::Finder<'static>; 4]> =
        LazyLock::new(|| GENERATED_MARKERS.map(memmem::Finder::new));

// Measured over 280 labeled files. Accuracy at 8KB equals the whole file on every shipped contest,
// and 1KB loses the evidence sitting under license headers.
const IDENTIFICATION_BYTES : usize = 8_192;
const NOT_CODE_MARKER_LINES : usize = 8;

const NO_SLOT : u16 = u16::MAX;

// The four kinds of declared symbol, as indices into the per-kind arrays of a scan
const STRINGS    : u8 = 0;
const COMMENTS   : u8 = 1;
const COM_STARTS : u8 = 2;
const COM_ENDS   : u8 = 3;

// What one side of a string pair may do. An ordinary quote is both sides in one symbol; a pair
// whose halves differ gets one slot per half, and its opener cannot close nor its closer open.
// 'RAW' is 'EITHER' without the backslash rule, for a symbol that serves as both ends of a form
// that escapes nothing: Go's and Odin's backtick against JavaScript's template literal.
// 'LITERAL' only exists as a pair on one line: the scan emits both halves or neither, which is
// what keeps a lifetime's lone ' from opening anything.
const ROLE_EITHER  : u8 = 0;
const ROLE_OPEN    : u8 = 1;
const ROLE_CLOSE   : u8 = 2;
const ROLE_LITERAL : u8 = 3;
const ROLE_RAW     : u8 = 4;
// A raw symbol with the language's escape character in front of it. Inside such a string the escape
// is an ordinary byte, so it may still close one; outside, it is an escape like any other and may
// not open one. Shell is where the two answers differ: 'echo I\'m done' writes one apostrophe.
const ROLE_RAW_ESCAPED : u8 = 5;

pub(crate) struct FileReport {
    pub shell: FileStats,
    pub sections: Vec<SectionReport>,
    pub bytes: usize,
}

pub(crate) struct SectionReport {
    pub language: String,
    pub stats: FileStats,
    pub bytes: usize,
}

impl FileReport {
    pub(crate) fn total_lines(&self) -> usize {
        self.shell.lines + self.sections.iter().map(|section| section.stats.lines).sum::<usize>()
    }

    // The whole file as one number, which is what the shell language's row shows: a container file
    // weighs all of its lines. The keywords stay the shell's own, because a section's keywords
    // belong to the section's language and are carried by the sections themselves.
    pub(crate) fn into_whole(mut self) -> FileStats {
        for section in &self.sections {
            self.shell.lines += section.stats.lines;
            self.shell.classes.add(&section.stats.classes);
        }
        self.shell
    }
}

pub(crate) enum FileOutcome {
    Counted(FileReport, Option<Arc<str>>),
    Skipped(ScanSkip)
}

pub(crate) fn parse_file(path: &Path, size: u64, lang_name: &str, buf: &mut Vec<u8>, buffers: &mut ParseBuffers,
    lookup: &NestedLanguageLookup, matchers: &mut KeywordMatchers,
    id_matchers: &mut IdentificationMatchers, config: &EngineConfig,
    written_by_hand: bool, extension_rules: Option<&crate::engine::identity::ExtensionRules>,
    shebang_map: &HashMap<String, Arc<str>>)
-> Result<FileOutcome,String>
{
    // None unless MEZURA_PHASE_TIMING is set, so a normal run never reads the clock at all
    let mut at = phase_timing::ENABLED.then(Instant::now);

    let mut file = match File::open(path){
        Ok(f) => f,
        Err(x) => return Err(x.to_string())
    };
    if let Some(t) = at {
        buffers.timing.open_nanos += phase_timing::nanos_since(t);
        at = Some(Instant::now());
    }

    let filled = match read_file_into(&mut file, buf, size) {
        Ok(filled) => filled,
        Err(x) => return Err(x.to_string())
    };
    if let Some(t) = at {
        buffers.timing.read_nanos += phase_timing::nanos_since(t);
        buffers.timing.bytes += filled as u64;
        buffers.timing.files += 1;
        at = Some(Instant::now());
    }
    // The wording is the one 'read_to_string' uses, so that the list of faulty files reads the same
    let Ok(contents) = str::from_utf8(&buf[..filled]) else {
        return Err("stream did not contain valid UTF-8".to_owned());
    };

    // Before the parse, which is what the skip saves: a bundle is the most expensive file there is.
    // A file the user named is counted whatever it holds, the same way the directory scan counts a
    // named file that a '.gitignore' covers.
    if !written_by_hand && let Some(kind) = find_scan_skip(contents, extension_rules, config) {
        return Ok(FileOutcome::Skipped(kind));
    }

    let resolved = extension_rules.and_then(|rules| rules.contenders.as_deref())
            .and_then(|c| identify_language(contents, c, lookup.languages, shebang_map, id_matchers))
            .map(|(name, _)| name);
    let lang_name = resolved.as_deref().unwrap_or(lang_name);
    let report = parse_lines::<false>(contents, lookup.languages.get(lang_name).unwrap(), lookup, matchers,
            config, buffers, &mut ExplainLog::default());
    if let Some(t) = at { buffers.timing.parse_nanos += phase_timing::nanos_since(t); }

    Ok(FileOutcome::Counted(report, resolved))
}

// What '--explain' calls: one file, read exactly as a counting run reads it, with the log switched
// on. Keywords are skipped whatever the configuration says, since they cannot move a line's class.
pub(crate) fn explain_parsed_file(contents: String, lang_name: &str, lookup: &NestedLanguageLookup,
    config: &EngineConfig) -> (String, FileReport, ExplainLog)
{
    let config = EngineConfig { count_keywords: false, ..config.clone() };
    let mut log = ExplainLog::default();
    let report = parse_lines::<true>(&contents, lookup.languages.get(lang_name).unwrap(), lookup,
            &mut KeywordMatchers::default(), &config, &mut ParseBuffers::default(), &mut log);
    (contents, report, log)
}

// Asking for one byte past the listed size tells a file of that size apart from one that grew.
// A unix read moves at most 2 GB at a time, so a short read is the end only once the listed size
// is reached. With no listed size the loop reads until a read returns nothing.
fn read_file_into(file: &mut File, buf: &mut Vec<u8>, size: u64) -> std::io::Result<usize> {
    const READ_WINDOW_BYTES : usize = 8_192;

    let expected = usize::try_from(size).unwrap_or(0);
    let mut filled = 0;
    loop {
        let end = if filled <= expected {expected + 1} else {filled + READ_WINDOW_BYTES};
        if buf.len() < end {
            buf.resize(end, 0);
        }
        match file.read(&mut buf[filled..end]) {
            Ok(0) => return Ok(filled),
            Ok(read) if filled + read < end && expected > 0 && filled + read >= expected
                    => return Ok(filled + read),
            Ok(read) => filled += read,
            Err(x) if x.kind() == std::io::ErrorKind::Interrupted => (),
            Err(x) => return Err(x)
        }
    }
}

// One declared symbol. 'next' chains every symbol that begins with the same byte, longest first,
// so that a '"""' is recognised before the '"' that starts it.
//
// 'anchor' is how far behind the searched byte the symbol begins, and 'anchors_of' below is where
// the choice of that byte is made.
// 'filler' is zero for every ordinary symbol. For a leveled one it is the counted byte: the slot's
// bytes are the prefix, and a match must find a run of the filler after it, then 'suffix'.
#[derive(Debug, Clone, Copy)]
struct Slot {
    symbol: u8,
    kind: u8,
    role: u8,
    len: u8,
    second: u8,
    anchor: u8,
    filler: u8,
    suffix: u8,
    cancelled_by: u8,
    next: u16,
}

#[derive(Debug, Clone, Copy)]
struct Chunk {
    bytes: [u8; 3],
    len: u8,
}

struct PlanEntry {
    kind: u8,
    symbol: u8,
    role: u8,
    filler: u8,
    suffix: u8,
    cancelled_by: u8,
    bytes: Box<[u8]>,
}

impl PlanEntry {
    fn of(kind: u8, symbol: u8, role: u8, bytes: &[u8]) -> PlanEntry {
        PlanEntry { kind, symbol, role, filler: 0, suffix: 0, cancelled_by: 0, bytes: bytes.into() }
    }

    fn leveled(kind: u8, symbol: u8, prefix: &[u8], suffix: u8) -> PlanEntry {
        PlanEntry { kind, symbol, role: ROLE_EITHER, filler: b'=', suffix, cancelled_by: 0,
                bytes: prefix.into() }
    }
}

// memchr searches up to three bytes in a single SIMD pass, so symbols are grouped by the byte that
// finds them and the groups packed into as few passes as the language allows: one for most, two for
// the handful with more than three distinct such bytes. A pass yields candidate positions, and only
// there is the rest of a symbol compared.
#[derive(Debug, Clone)]
pub(crate) struct ScanPlan {
    chunks: Vec<Chunk>,
    first: [u16; 256],
    slots: Vec<Slot>,
    symbols: Vec<Box<[u8]>>,
    sorted_kinds: [bool; 4],
    // Whether a line opening with a line comment symbol is a comment and nothing else. False where a
    // block opener begins with one, as Lua's '--[[' begins with '--', CMake's '#[[' and Julia's '#='
    // with '#': there the same bytes open a block that runs on past this line.
    line_comment_ends_the_line: bool,
}

impl ScanPlan {
    pub(crate) fn build(language: &Language) -> ScanPlan {
        // The single line symbols first, the character literals after them and the crossing ones
        // last, which is the numbering 'Language::get_string_pair_of' answers to
        let mut entries : Vec<PlanEntry> = Vec::new();
        let (symbols, literals) = (language.strings.get_symbols(), language.strings.get_char_literals());
        for (i, symbol) in symbols.iter().enumerate() {
            entries.push(PlanEntry::of(STRINGS, i as u8, ROLE_EITHER, symbol.as_bytes()));
        }
        for (i, symbol) in literals.iter().enumerate() {
            let index = (symbols.len() + i) as u8;
            entries.push(PlanEntry::of(STRINGS, index, ROLE_LITERAL, symbol.as_bytes()));
        }
        for (i, crossing) in language.strings.get_multiline_strings().iter().enumerate() {
            let index = (symbols.len() + literals.len() + i) as u8;
            let (open, close) = (&crossing.open, &crossing.close);
            if open != close {
                entries.push(PlanEntry::of(STRINGS, index, ROLE_OPEN, open.as_bytes()));
                entries.push(PlanEntry::of(STRINGS, index, ROLE_CLOSE, close.as_bytes()));
            } else {
                let role = if crossing.escapes {ROLE_EITHER} else {ROLE_RAW};
                entries.push(PlanEntry::of(STRINGS, index, role, open.as_bytes()));
            }
        }
        for (i, symbol) in language.comment_symbols.iter().enumerate() {
            entries.push(PlanEntry::of(COMMENTS, i as u8, ROLE_EITHER, symbol.as_bytes()));
        }
        // Numbered by the language itself, so this and the helpers that answer to those numbers
        // cannot disagree about the order. A leveled slot holds its prefix as the bytes.
        for (i, pair) in language.comment_pairs().enumerate() {
            let index = i as u8;
            match pair {
                CommentPair::Plain { start, end } | CommentPair::Nesting { start, end } => {
                    entries.push(PlanEntry::of(COM_STARTS, index, ROLE_EITHER, start.as_bytes()));
                    entries.push(PlanEntry::of(COM_ENDS, index, ROLE_EITHER, end.as_bytes()));
                },
                CommentPair::Leveled(pair) => {
                    entries.push(PlanEntry::leveled(COM_STARTS, index, pair.start_prefix.as_bytes(), pair.start_suffix));
                    entries.push(PlanEntry::leveled(COM_ENDS, index, pair.end_prefix.as_bytes(), pair.end_suffix));
                }
            }
        }
        for (symbol, cancelling) in &language.cancelled_symbols {
            for entry in entries.iter_mut().filter(|entry| *entry.bytes == *symbol.as_bytes()) {
                entry.cancelled_by = *cancelling;
            }
        }
        entries.retain(|entry| !entry.bytes.is_empty());
        let line_comment_ends_the_line = !entries.iter().filter(|entry| entry.kind == COM_STARTS)
                .any(|start| entries.iter().filter(|entry| entry.kind == COMMENTS)
                        .any(|comment| start.bytes.starts_with(&comment.bytes)));
        entries.sort_by_key(|entry| std::cmp::Reverse(entry.bytes.len()));

        let anchors = anchors_of(&entries);
        let mut first = [NO_SLOT; 256];
        let (mut slots, mut symbols) = (Vec::with_capacity(entries.len()), Vec::with_capacity(entries.len()));
        for (entry, anchor) in entries.iter().zip(&anchors) {
            let index = slots.len() as u16;
            let anchor = *anchor;
            slots.push(Slot {
                symbol: entry.symbol,
                kind: entry.kind,
                role: entry.role,
                len: entry.bytes.len() as u8,
                second: if entry.bytes.len() > 1 { entry.bytes[1] } else { 0 },
                anchor,
                filler: entry.filler,
                suffix: entry.suffix,
                cancelled_by: entry.cancelled_by,
                next: NO_SLOT,
            });
            symbols.push(entry.bytes.clone());
            let head = &mut first[entry.bytes[anchor as usize] as usize];
            if *head == NO_SLOT {
                *head = index;
            } else {
                let mut cursor = *head as usize;
                while slots[cursor].next != NO_SLOT { cursor = slots[cursor].next as usize }
                slots[cursor].next = index;
            }
        }

        let (chunks, mut sorted_kinds) = pack_into_chunks(&entries, &anchors);
        // An anchored match begins behind the byte that found it, so a kind holding symbols
        // anchored at two different depths reports them out of line order and is sorted afterwards.
        for (kind, sorted) in sorted_kinds.iter_mut().enumerate() {
            let mut depths = entries.iter().zip(&anchors)
                    .filter(|(entry, _)| entry.kind as usize == kind).map(|(_, anchor)| *anchor);
            let Some(first) = depths.next() else { continue };
            if depths.any(|depth| depth != first) { *sorted = true }
        }
        ScanPlan { chunks, first, slots, symbols, sorted_kinds, line_comment_ends_the_line }
    }
}

// Where in each symbol the byte that finds it sits. The bytes are not equally cheap: a letter is
// visited on every word of the file, and any byte at all costs a memchr pass over every line once
// three others are already spoken for. So the fewest bytes that reach every symbol are chosen, and
// each symbol is anchored on the first of its own bytes among them.
//
// 'r#"' and 'R"(' begin with a letter and are found by the quote their language declares anyway.
// ')"' begins with a bracket, which stands in front of every call in C++, and is found by that
// same quote.
fn anchors_of(entries: &[PlanEntry]) -> Vec<u8> {
    // A symbol offering one byte has no say in the matter: that byte is searched either way
    let mut searched : Vec<u8> = Vec::new();
    for entry in entries {
        let mut bytes = get_candidate_bytes_of(entry);
        let Some(first) = bytes.next() else { continue };
        if bytes.all(|byte| byte == first) && !searched.contains(&first) { searched.push(first) }
    }
    while let Some(byte) = find_the_byte_reaching_most_of(entries, &searched) {
        searched.push(byte);
    }

    // A symbol of nothing but letters reaches none of the searched bytes and keeps its first
    entries.iter().map(|entry| entry.bytes.iter().position(|byte| searched.contains(byte))
            .unwrap_or(0) as u8).collect()
}

fn get_candidate_bytes_of(entry: &PlanEntry) -> impl Iterator<Item = u8> + '_ {
    entry.bytes.iter().copied().filter(|byte| !byte.is_ascii_alphanumeric())
}

fn is_reached_by(entry: &PlanEntry, searched: &[u8]) -> bool {
    get_candidate_bytes_of(entry).any(|byte| searched.contains(&byte))
}

// On a tie the byte met first wins, so a language's plan comes out the same on every run.
fn find_the_byte_reaching_most_of(entries: &[PlanEntry], searched: &[u8]) -> Option<u8> {
    let waiting = entries.iter().filter(|entry| get_candidate_bytes_of(entry).next().is_some()
            && !is_reached_by(entry, searched)).collect::<Vec<&PlanEntry>>();

    let mut best : Option<(u8, usize)> = None;
    for entry in &waiting {
        for byte in get_candidate_bytes_of(entry) {
            let reach = waiting.iter().filter(|other|
                    get_candidate_bytes_of(other).any(|other_byte| other_byte == byte)).count();
            if best.is_none_or(|(_, most)| reach > most) { best = Some((byte, reach)) }
        }
    }
    best.map(|(byte, _)| byte)
}

// Two kinds sharing a first byte must be searched in the same pass, or that byte gets visited twice.
// Kinds are grouped by that overlap and the groups packed whole, which is what leaves every output
// vector already in the order the positions appear on the line. A group of more than three distinct
// bytes cannot be one pass, so it is split and its kinds are marked as needing a sort after all.
fn pack_into_chunks(entries: &[PlanEntry], anchors: &[u8]) -> (Vec<Chunk>, [bool; 4]) {
    let mut bytes_of_kind : [Vec<u8>; 4] = Default::default();
    for (entry, anchor) in entries.iter().zip(anchors) {
        let searched = entry.bytes[*anchor as usize];
        let set = &mut bytes_of_kind[entry.kind as usize];
        if !set.contains(&searched) { set.push(searched) }
    }

    let mut group_of = [0usize, 1, 2, 3];
    for a in 0..4 {
        for b in (a + 1)..4 {
            if bytes_of_kind[a].iter().any(|x| bytes_of_kind[b].contains(x)) {
                let (from, to) = (group_of[b], group_of[a]);
                for slot in group_of.iter_mut() { if *slot == from { *slot = to } }
            }
        }
    }

    let (mut chunks, mut sorted_kinds) : (Vec<Vec<u8>>, [bool; 4]) = (Vec::new(), [false; 4]);
    for group in 0..4 {
        let mut group_bytes : Vec<u8> = Vec::new();
        for kind in 0..4 {
            if group_of[kind] != group { continue }
            for byte in &bytes_of_kind[kind] {
                if !group_bytes.contains(byte) { group_bytes.push(*byte) }
            }
        }
        if group_bytes.is_empty() { continue }

        if group_bytes.len() > 3 {
            for kind in 0..4 { if group_of[kind] == group { sorted_kinds[kind] = true } }
            for piece in group_bytes.chunks(3) { chunks.push(piece.to_vec()) }
            continue;
        }
        match chunks.iter_mut().find(|chunk| chunk.len() + group_bytes.len() <= 3) {
            Some(chunk) => chunk.extend_from_slice(&group_bytes),
            None => chunks.push(group_bytes)
        }
    }

    let chunks = chunks.into_iter().map(|bytes| {
        let mut padded = [0u8; 3];
        padded[..bytes.len()].copy_from_slice(&bytes);
        Chunk { bytes: padded, len: bytes.len() as u8 }
    }).collect();

    (chunks, sorted_kinds)
}

fn get_or_build_plan_of(language: &Language) -> &ScanPlan {
    language.scan_plan.get_or_init(|| ScanPlan::build(language))
}

// The per line working memory, owned by the consumer thread and cleared rather than reallocated.
#[derive(Debug, Default)]
pub(crate) struct ScanBuffers {
    raw_strings: Vec<(usize, u8, u8)>,
    strings: Vec<usize>,
    string_symbols: Vec<u8>,
    comments: Vec<usize>,
    // Position, pair, and the level a leveled occurrence carried, zero for every other pair
    com_starts: Vec<(usize, u8, u8)>,
    com_ends: Vec<(usize, u8, u8)>,
    consumed: Vec<usize>,
    // Offsets into the line, where 'ParseBuffers::code_spans' holds offsets into the whole file
    code_ranges: Vec<(usize, usize)>,
}

impl ScanBuffers {
    fn reset(&mut self, slots: usize) {
        self.raw_strings.clear();
        self.strings.clear();
        self.string_symbols.clear();
        self.comments.clear();
        self.com_starts.clear();
        self.com_ends.clear();
        self.consumed.clear();
        self.consumed.resize(slots, 0);
        self.code_ranges.clear();
    }
}

// The keyword scratch cannot live in ScanBuffers: while a LineInfo borrows the cleansed line out of
// it, the whole struct is borrowed, and counting the keywords of that very line needs a free one.
#[derive(Debug, Default)]
pub(crate) struct ParseBuffers {
    scan: ScanBuffers,
    alias_indices: Vec<usize>,
    // every stretch of the file that is code, gathered line by line so that the keywords can be
    // searched once over the whole buffer instead of once per alias per line
    code_spans: Vec<(u32, u32)>,
    pub timing: phase_timing::Totals,
}

// A comment symbol spelled with letters is a word and not a prefix: Batch opens a comment with REM,
// and REMOVE is a command that has to count as one. Only a symbol whose own ends are word characters
// asks the question at all, so '//' and '#' pay nothing for it.
fn stands_as_its_own_word(line: &[u8], start: usize, width: usize) -> bool {
    let is_word = |byte: u8| byte.is_ascii_alphanumeric() || byte == b'_';
    let glued_before = is_word(line[start]) && start > 0 && is_word(line[start - 1]);
    let glued_after = is_word(line[start + width - 1])
            && line.get(start + width).is_some_and(|byte| is_word(*byte));

    !glued_before && !glued_after
}

// An even run of the language's escape character in front of a symbol leaves it standing, since each
// pair escapes itself. A language that declares none escapes nothing anywhere.
fn is_not_escaped(pos: usize, bytes: &[u8], escape: Option<u8>) -> bool {
    let Some(escape) = escape else { return true };
    let mut escapes = 0;
    let mut offset = 1;
    while pos >= offset && bytes[pos - offset] == escape {
        offset += 1;
        escapes += 1;
    }
    escapes % 2 == 0
}

fn scan_line(line: &str, language: &Language, buffers: &mut ScanBuffers) {
    let plan = get_or_build_plan_of(language);
    let line_bytes = line.as_bytes();
    let escape = language.strings.get_escape();
    buffers.reset(plan.slots.len());

    for chunk in &plan.chunks {
        match chunk.len {
            1 => for at in memchr::memchr_iter(chunk.bytes[0], line_bytes) {
                take_symbols_at(at, line_bytes, plan, buffers, escape)
            },
            2 => for at in memchr::memchr2_iter(chunk.bytes[0], chunk.bytes[1], line_bytes) {
                take_symbols_at(at, line_bytes, plan, buffers, escape)
            },
            _ => for at in memchr::memchr3_iter(chunk.bytes[0], chunk.bytes[1], chunk.bytes[2], line_bytes) {
                take_symbols_at(at, line_bytes, plan, buffers, escape)
            }
        }
    }

    // Only a kind split across two passes reaches here: one pass yields its symbols in line order
    if plan.sorted_kinds[STRINGS as usize] {
        let length_of = |symbol: u8, role: u8| {
            let (open, close) = language.get_string_pair_of(symbol);
            match role { ROLE_CLOSE => close.len(), _ => open.len() }
        };
        buffers.raw_strings.sort_unstable_by(|(a_at, a_symbol, a_role), (b_at, b_symbol, b_role)|
                a_at.cmp(b_at).then_with(|| length_of(*b_symbol, *b_role).cmp(&length_of(*a_symbol, *a_role))));
    }
    if plan.sorted_kinds[COMMENTS as usize] { buffers.comments.sort_unstable() }
    if plan.sorted_kinds[COM_STARTS as usize] {
        buffers.com_starts.sort_unstable_by(|(a_at, a_symbol, a_level), (b_at, b_symbol, b_level)|
                a_at.cmp(b_at).then_with(|| language.comment_start_len(*b_symbol, *b_level)
                        .cmp(&language.comment_start_len(*a_symbol, *a_level))));
    }
    if plan.sorted_kinds[COM_ENDS as usize] {
        buffers.com_ends.sort_unstable_by(|(a_at, a_symbol, a_level), (b_at, b_symbol, b_level)|
                a_at.cmp(b_at).then_with(|| language.comment_end_len(*b_symbol, *b_level)
                        .cmp(&language.comment_end_len(*a_symbol, *a_level))));
    }
}

fn take_symbols_at(at: usize, line_bytes: &[u8], plan: &ScanPlan, buffers: &mut ScanBuffers,
    escape: Option<u8>)
{
    let mut cursor = plan.first[line_bytes[at] as usize];
    while cursor != NO_SLOT {
        let index = cursor as usize;
        let slot = plan.slots[index];
        cursor = slot.next;

        let Some(start) = at.checked_sub(slot.anchor as usize) else { continue };
        // Each symbol is searched without overlapping itself, so "///" holds one "//" and not two.
        // A counted slot is exempt: every level shares the one slot, so ']]' and ']=]' are two
        // different symbols rather than one overlapping itself, and the shorter would hide the
        // longer that begins inside it.
        if slot.filler == 0 && start < buffers.consumed[index] { continue }
        let matched = match (slot.anchor, slot.len) {
            (0, 1) if slot.filler == 0 => true,
            (0, 2) if slot.filler == 0 => line_bytes.get(at + 1) == Some(&slot.second),
            _ => line_bytes[start..].starts_with(&plan.symbols[index])
        };
        if !matched { continue }
        // Where the language writes the symbol and the character before it as one longer form, the
        // symbol is part of that form and not itself. C3's '<*' opens a documentation comment
        // everywhere except in 'int[<*>]', which is a vector of unknown length.
        if slot.cancelled_by != 0 && start > 0 && line_bytes[start - 1] == slot.cancelled_by {
            continue;
        }
        // The level is carried beside the position, so only an end with the same count answers it
        let mut level = 0u8;
        let mut width = slot.len as usize;
        if slot.filler != 0 {
            let mut cursor = start + slot.len as usize;
            while line_bytes.get(cursor) == Some(&slot.filler) && level < u8::MAX {
                cursor += 1;
                level += 1;
            }
            if line_bytes.get(cursor) != Some(&slot.suffix) { continue }
            width = cursor + 1 - start;
        }
        // An escape cancels a string symbol and nothing else. An escaped raw symbol is kept and
        // marked instead of dropped, since whether it counts depends on something the scan cannot
        // see: the resolution below knows whether it would open a string or close one.
        let mut role = slot.role;
        if slot.kind == STRINGS && start != 0 && !is_not_escaped(start, line_bytes, escape) {
            match slot.role {
                ROLE_EITHER | ROLE_LITERAL => continue,
                ROLE_RAW => role = ROLE_RAW_ESCAPED,
                _ => ()
            }
        }

        // Whatever sits between the two halves is inside the taken pair, so the resolution below
        // drops it on its own.
        if slot.role == ROLE_LITERAL {
            let symbol_bytes = &plan.symbols[index];
            let mut cursor = start + width;
            let closed_at = loop {
                let Some(offset) = memchr::memchr(symbol_bytes[0], &line_bytes[cursor..]) else { break None };
                let candidate = cursor + offset;
                if line_bytes[candidate..].starts_with(symbol_bytes)
                        && is_not_escaped(candidate, line_bytes, escape)
                        && holds_one_character(&line_bytes[start + width..candidate]) {
                    break Some(candidate);
                }
                cursor = candidate + 1;
            };
            let Some(closed_at) = closed_at else {
                buffers.consumed[index] = start + width;
                continue;
            };
            buffers.raw_strings.push((start, slot.symbol, ROLE_OPEN));
            buffers.raw_strings.push((closed_at, slot.symbol, ROLE_CLOSE));
            buffers.consumed[index] = closed_at + width;
            continue;
        }

        buffers.consumed[index] = start + width;
        match slot.kind {
            STRINGS => buffers.raw_strings.push((start, slot.symbol, role)),
            COMMENTS => if stands_as_its_own_word(line_bytes, start, width) {
                buffers.comments.push(start);
            },
            COM_STARTS => buffers.com_starts.push((start, slot.symbol, level)),
            _ => buffers.com_ends.push((start, slot.symbol, level))
        }
    }
}

#[derive(Debug)]
pub(crate) struct IdentificationMatcher {
    rules: Vec<(memmem::Finder<'static>, String, bool)>,
}

impl IdentificationMatcher {
    pub(crate) fn build(language: &Language) -> Option<IdentificationMatcher> {
        Self::of(&language.identifying_line_starts, &language.identifying_line_contains)
    }

    pub(crate) fn of(line_starts: &[String], line_contains: &[String]) -> Option<IdentificationMatcher> {
        let rule = |literal: &String, at_line_start: bool|
                (memmem::Finder::new(literal.as_str()).into_owned(), literal.clone(), at_line_start);
        let rules = line_starts.iter().filter(|x| !x.is_empty()).map(|x| rule(x, true))
                .chain(line_contains.iter().filter(|x| !x.is_empty()).map(|x| rule(x, false)))
                .collect::<Vec<_>>();
        if rules.is_empty() {None} else {Some(IdentificationMatcher { rules })}
    }

    pub(crate) fn find_evidence(&self, head: &[u8]) -> Option<(usize, &str)> {
        self.rules.iter().filter_map(|(finder, literal, at_line_start)| {
            let mut from = 0;
            while let Some(offset) = finder.find(&head[from..]) {
                let at = from + offset;
                from = at + 1;
                if evidence_stands(head, at, literal.len(), *at_line_start, false) {
                    return Some((at, literal.as_str()));
                }
            }
            None
        }).min_by_key(|(at, _)| *at)
    }

    // A line-start marker may run into a longer word ('-keep' catches '-keepnames') and reads the
    // deeper byte window, since a directive can sit behind a long comment header. A contains marker
    // keeps the word boundary and is believed only on the top lines, where a dependency file's
    // rules sit and a stray token inside real code rarely does.
    pub(crate) fn finds_a_marker(&self, buf: &str) -> bool {
        let head = head_of(buf);
        let lines = first_lines_of(buf, NOT_CODE_MARKER_LINES);
        self.rules.iter().any(|(finder, literal, at_line_start)| {
            let hay = if *at_line_start {head} else {lines};
            let mut from = 0;
            while let Some(offset) = finder.find(&hay[from..]) {
                let at = from + offset;
                from = at + 1;
                if evidence_stands(hay, at, literal.len(), *at_line_start, *at_line_start) {
                    return true;
                }
            }
            false
        })
    }
}

// A literal must not run into a word on either side, so 'class' passes over 'classic_t'.
fn evidence_stands(head: &[u8], at: usize, len: usize, at_line_start: bool, allows_a_word_after: bool) -> bool {
    let is_word = |b: u8| b.is_ascii_alphanumeric() || b == b'_';
    if len == 0 {
        return false;
    }
    if !allows_a_word_after && is_word(head[at + len - 1]) && head.get(at + len).copied().is_some_and(is_word) {
        return false;
    }
    if is_word(head[at]) && at > 0 && is_word(head[at - 1]) {
        return false;
    }
    if !at_line_start {
        return true;
    }
    head[..at].iter().rev().take_while(|b| **b != b'\n').all(|b| matches!(b, b' ' | b'\t' | b'\r'))
}

#[derive(Default)]
pub(crate) struct IdentificationMatchers {
    by_language: HashMap<String, Option<IdentificationMatcher>>,
}

impl IdentificationMatchers {
    fn for_language(&mut self, language: &Language) -> Option<&IdentificationMatcher> {
        self.by_language.entry(language.name.clone())
                .or_insert_with(|| IdentificationMatcher::build(language)).as_ref()
    }
}

pub(crate) fn find_identified_language(buf: &str, contenders: &[Arc<str>],
        languages: &HashMap<String, Language>, shebang_map: &HashMap<String, Arc<str>>)
-> Option<(Arc<str>, String, usize)>
{
    let head = head_of(buf);
    let (name, offset) = identify_language(buf, contenders, languages, shebang_map,
            &mut IdentificationMatchers::default())?;
    match offset {
        None => Some((name, String::from_utf8_lossy(first_line_of(head)).trim_end().to_owned(), 1)),
        Some(offset) => {
            let literal = languages.get(name.as_ref()).and_then(IdentificationMatcher::build)
                    .and_then(|matcher| matcher.find_evidence(head).map(|(_, x)| x.to_owned()))?;
            let line = head[..offset].iter().filter(|b| **b == b'\n').count() + 1;
            Some((name, literal, line))
        }
    }
}

// The earliest evidence in the file wins and ties keep the standing order, so an '@interface' on
// line 1 is not beaten by a lone '%' inside a later comment. A '#!' answer carries no offset.
fn identify_language(buf: &str, contenders: &[Arc<str>], languages: &HashMap<String, Language>,
        shebang_map: &HashMap<String, Arc<str>>, matchers: &mut IdentificationMatchers)
-> Option<(Arc<str>, Option<usize>)>
{
    let head = head_of(buf);
    if let Some(name) = find_shebang_language(head, shebang_map) {
        return Some((name, None));
    }
    let mut best: Option<(usize, usize)> = None;
    for (at, name) in contenders.iter().enumerate() {
        if let Some(language) = languages.get(name.as_ref())
            && let Some(matcher) = matchers.for_language(language)
            && let Some((offset, _)) = matcher.find_evidence(head)
            && best.is_none_or(|(earliest, _)| offset < earliest) {
            best = Some((offset, at));
        }
    }
    best.map(|(offset, at)| (contenders[at].clone(), Some(offset)))
}

// Through the run's own interpreter map rather than the raw declarations, so a forced interpreter
// answers a contested file and an extensionless one identically.
fn find_shebang_language(head: &[u8], shebang_map: &HashMap<String, Arc<str>>) -> Option<Arc<str>> {
    let token = crate::engine::identity::find_interpreter(first_line_of(head))?;
    crate::engine::identity::find_language_of_interpreter(shebang_map, str::from_utf8(token).ok()?)
}

fn head_of(buf: &str) -> &[u8] {
    let bytes = strip_bom(buf.as_bytes());
    &bytes[..bytes.len().min(IDENTIFICATION_BYTES)]
}

// Uncapped on purpose. A cargo dependency file writes each rule on one line, which can run tens of
// kilobytes past the identification window, and its artifact rule sits on the third line or later.
fn first_lines_of(buf: &str, lines: usize) -> &[u8] {
    let bytes = strip_bom(buf.as_bytes());
    let mut from = 0;
    for _ in 0..lines {
        match memchr::memchr(b'\n', &bytes[from..]) {
            Some(at) => from += at + 1,
            None => return bytes
        }
    }
    &bytes[..from]
}

// A byte order mark is not whitespace, so left in place it defeats every line-start rule.
fn strip_bom(bytes: &[u8]) -> &[u8] {
    bytes.strip_prefix(b"\xef\xbb\xbf".as_slice()).unwrap_or(bytes)
}

fn first_line_of(head: &[u8]) -> &[u8] {
    &head[..memchr::memchr(b'\n', head).unwrap_or(head.len())]
}

pub(crate) struct KeywordMatcher {
    aliases_with_indices: Vec<(memmem::Finder<'static>, usize, usize)>,
}

impl KeywordMatcher {
    pub(crate) fn build(language: &Language) -> Option<KeywordMatcher> {
        let mut aliases_with_indices = Vec::new();
        for (keyword_index, keyword) in language.keywords.iter().enumerate() {
            for alias in &keyword.aliases {
                aliases_with_indices.push((memmem::Finder::new(alias.as_str()).into_owned(), alias.len(), keyword_index));
            }
        }
        if aliases_with_indices.is_empty() {
            None
        } else {
            Some(KeywordMatcher { aliases_with_indices })
        }
    }
}

// 'extension_to_name' and 'set_aside' cover the whole shipped set even when a run narrowed its
// languages, so that '--languages vue' still knows what JavaScript is; a caller with no sections
// in play hands empty maps.
pub(crate) struct NestedLanguageLookup<'a> {
    pub languages: &'a HashMap<String, Language>,
    pub extension_to_name: &'a HashMap<String, std::sync::Arc<str>>,
    pub set_aside: &'a HashMap<String, Language>,
}

impl NestedLanguageLookup<'_> {
    // What a tag says its section is written in, which people write either way: 'lang="scss"' is an
    // extension and 'type="text/typescript"' is a language's name. The extension is tried first,
    // since it is the form the declared defaults use and the one the user's priority rules answer
    // for, and the name after it, so a language whose name is a whole word is found by that word.
    fn find_by_spelling(&self, spelling: &str) -> Option<&Language> {
        let lowered = spelling.to_lowercase();
        if let Some(name) = self.extension_to_name.get(&lowered) {
            return self.find_by_name(name.as_ref());
        }
        self.languages.values().chain(self.set_aside.values())
                .find(|language| language.name.to_lowercase() == lowered)
    }

    pub(crate) fn find_by_name(&self, name: &str) -> Option<&Language> {
        self.languages.get(name).or_else(|| self.set_aside.get(name))
    }
}

#[derive(Default)]
pub(crate) struct KeywordMatchers {
    by_language: HashMap<String, Option<KeywordMatcher>>,
}

impl KeywordMatchers {
    fn for_language(&mut self, language: &Language) -> Option<&KeywordMatcher> {
        self.by_language.entry(language.name.clone())
                .or_insert_with(|| KeywordMatcher::build(language)).as_ref()
    }
}

// The order decides which reason a file that trips two of the checks is reported under
pub(crate) fn find_scan_skip(contents: &str, rules: Option<&crate::engine::identity::ExtensionRules>,
        config: &EngineConfig) -> Option<ScanSkip> {
    if !config.count_not_code && rules.and_then(|x| x.not_code.as_ref())
            .is_some_and(|matcher| matcher.finds_a_marker(contents)) {
        return Some(ScanSkip::NotCode);
    }
    if !config.count_minified && is_minified(contents) {
        return Some(ScanSkip::Minified);
    }
    if !config.count_generated && is_generated(contents) {
        return Some(ScanSkip::Generated);
    }
    None
}

// The average and not where the first break falls. Webpack writes a licence comment on line one
// and the payload on line two.
fn is_minified(contents: &str) -> bool {
    if contents.len() < SMALLEST_FILE_WORTH_TESTING {
        return false;
    }
    // Counting stops at the number of lines that settles the file, so an ordinary file of any size
    // is read for a few KB and a bundle to its end
    let most_lines = contents.len() / MINIFIED_AVERAGE_LINE_BYTES;
    memchr::memchr_iter(b'\n', contents.as_bytes()).take(most_lines).count() < most_lines
}

// The head, lowercased into a buffer of its own so the markers can be matched without case
fn is_generated(contents: &str) -> bool {
    let head = &contents.as_bytes()[..contents.len().min(GENERATED_MARKER_BYTES)];
    let mut lowercased = [0u8; GENERATED_MARKER_BYTES];
    lowercased[..head.len()].copy_from_slice(head);
    lowercased[..head.len()].make_ascii_lowercase();
    GENERATED_FINDERS.iter().any(|finder| finder.find(&lowercased[..head.len()]).is_some())
}

// The same lines 'str::lines' hands out, trailing '\r' dropped the same way, found with memchr's
// SIMD search instead of the standard library's word-at-a-time loop.
struct LineIter<'a> {
    contents: &'a str,
    newlines: memchr::Memchr<'a>,
    start: usize,
}

impl<'a> Iterator for LineIter<'a> {
    type Item = (usize, &'a str);

    fn next(&mut self) -> Option<(usize, &'a str)> {
        match self.newlines.next() {
            Some(at) => {
                let mut end = at;
                if end > self.start && self.contents.as_bytes()[end - 1] == b'\r' {
                    end -= 1;
                }
                let line = (self.start, &self.contents[self.start..end]);
                self.start = at + 1;
                Some(line)
            },
            None => {
                if self.start >= self.contents.len() {
                    return None;
                }
                let line = (self.start, &self.contents[self.start..]);
                self.start = self.contents.len();
                Some(line)
            }
        }
    }
}

fn get_lines_of(contents: &str) -> LineIter<'_> {
    LineIter { contents, newlines: memchr::memchr_iter(b'\n', contents.as_bytes()), start: 0 }
}

// The carry from one line to the next, one set per language in play: the shell's survives a
// section, a section's starts fresh at its opener and is dropped at its closer.
//
// 'opened_line' is written only while explaining: the line that opened whatever is currently
// carried. One slot covers comment, string and continuation because at most one of the three
// crosses a line boundary.
#[derive(Default)]
struct WalkState {
    open_comment: Option<(u8, u32)>,
    open_str_symbol: Option<u8>,
    continued_comment: bool,
    opened_line: usize,
}

// What earlier lines left open when a line began, as '--explain' reports it. For a leveled pair the
// depth carried from line to line is the level, so the spelling of the opener is rebuilt from it
// later. 'ends' says the carried thing is gone by the line's end: closed, or replaced by a new one
// of the same symbol that this line itself opened.
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) enum CarriedRecord {
    Nothing,
    Comment { symbol: u8, depth: u32, since_line: usize, ends: bool },
    Str { symbol: u8, since_line: usize, ends: bool },
    Continuation { since_line: usize },
}

impl CarriedRecord {
    fn of(state: &WalkState) -> CarriedRecord {
        if let Some(symbol) = state.open_str_symbol {
            CarriedRecord::Str { symbol, since_line: state.opened_line, ends: false }
        } else if let Some((symbol, depth)) = state.open_comment {
            CarriedRecord::Comment { symbol, depth, since_line: state.opened_line, ends: false }
        } else if state.continued_comment {
            CarriedRecord::Continuation { since_line: state.opened_line }
        } else {
            CarriedRecord::Nothing
        }
    }

    // Asked after the line has been read, against the state it left behind: the carried thing
    // survives only where the same kind is still open and was not opened anew on this line
    fn with_its_end_marked(self, state: &WalkState, opened_here: OpenedHere) -> CarriedRecord {
        match self {
            CarriedRecord::Comment { symbol, depth, since_line, .. } => CarriedRecord::Comment {
                symbol, depth, since_line,
                ends: state.open_comment.is_none() || opened_here.comment },
            CarriedRecord::Str { symbol, since_line, .. } => CarriedRecord::Str {
                symbol, since_line,
                ends: state.open_str_symbol.is_none() || opened_here.string },
            other => other,
        }
    }
}

// One verdict per physical line, in file order. The language of a record is interned, since a file
// rarely holds more than a few.
#[derive(Default)]
pub(crate) struct ExplainLog {
    records: Vec<LineRecord>,
    languages: Vec<String>,
}

pub(crate) struct LineRecord {
    pub class: LineClass,
    pub carried: CarriedRecord,
    // Byte offsets into the raw line as the file spells it, trimmed leading whitespace included
    pub spans: Vec<Span>,
    // An index into the interned names that 'into_parts' hands out beside the records
    pub language: u16,
}

impl ExplainLog {
    fn record(&mut self, class: LineClass, carried: CarriedRecord, language: &Language, spans: Vec<Span>) {
        let language = match self.languages.iter().position(|name| name == &language.name) {
            Some(at) => at as u16,
            None => {
                self.languages.push(language.name.clone());
                (self.languages.len() - 1) as u16
            }
        };
        self.records.push(LineRecord { class, carried, spans, language });
    }

    // The number of the line being walked right now, 1-based: its record has not been pushed yet
    fn get_current_line_number(&self) -> usize {
        self.records.len() + 1
    }

    #[cfg(test)]
    pub(crate) fn get_language_name_of(&self, record: &LineRecord) -> &str {
        &self.languages[record.language as usize]
    }

    #[cfg(test)]
    pub(crate) fn records(&self) -> &[LineRecord] {
        &self.records
    }

    pub(crate) fn into_parts(self) -> (Vec<LineRecord>, Vec<String>) {
        (self.records, self.languages)
    }
}

struct SectionBucket<'a> {
    language: &'a Language,
    stats: FileStats,
    spans: Vec<(u32, u32)>,
    collecting_spans: bool,
    bytes: usize,
}

// Every symbol a language declares begins at one of the bytes its scan plan searches, so a line
// holding none of them holds no symbol and needs no scan. One pass per chunk over the whole file
// answers that for every line at once, and the answers are taken lazily: the lines are asked in file
// order, so each pass only ever moves forward, whether or not the line before it asked.
struct CandidateProbe<'a> {
    passes: Vec<CandidatePass<'a>>,
}

enum CandidatePass<'a> {
    One(Peekable<memchr::Memchr<'a>>),
    Two(Peekable<memchr::Memchr2<'a>>),
    Three(Peekable<memchr::Memchr3<'a>>)
}

impl<'a> CandidateProbe<'a> {
    fn of(contents: &'a str, plan: &ScanPlan) -> CandidateProbe<'a> {
        let bytes = contents.as_bytes();
        let passes = plan.chunks.iter().map(|chunk| match chunk.len {
            1 => CandidatePass::One(memchr::memchr_iter(chunk.bytes[0], bytes).peekable()),
            2 => CandidatePass::Two(memchr::memchr2_iter(chunk.bytes[0], chunk.bytes[1], bytes).peekable()),
            _ => CandidatePass::Three(memchr::memchr3_iter(chunk.bytes[0], chunk.bytes[1], chunk.bytes[2], bytes).peekable())
        }).collect();
        CandidateProbe { passes }
    }

    // The range is the line as the file spells it, leading and trailing whitespace included, which
    // covers the trimmed line the scan would actually read
    fn has_a_candidate_in(&mut self, from: usize, to: usize) -> bool {
        self.passes.iter_mut().any(|pass| match pass {
            CandidatePass::One(pass) => reaches_into(pass, from, to),
            CandidatePass::Two(pass) => reaches_into(pass, from, to),
            CandidatePass::Three(pass) => reaches_into(pass, from, to)
        })
    }
}

fn reaches_into(pass: &mut Peekable<impl Iterator<Item = usize>>, from: usize, to: usize) -> bool {
    while pass.next_if(|at| *at < from).is_some() {}
    pass.peek().is_some_and(|at| *at < to)
}

fn parse_lines<const EXPLAIN: bool>(contents: &str, language: &Language, lookup: &NestedLanguageLookup,
    matchers: &mut KeywordMatchers, config: &EngineConfig, buffers: &mut ParseBuffers,
    log: &mut ExplainLog) -> FileReport
{
    let ParseBuffers { scan, alias_indices, code_spans, .. } = buffers;
    let mut shell_stats = if config.count_keywords { FileStats::with_keywords(&language.keywords) }
            else { FileStats::default() };
    code_spans.clear();

    // Nothing but the keyword search reads the spans, so a language with no keywords builds none
    let collecting_spans = config.count_keywords && matchers.for_language(language).is_some();

    let mut shell = WalkState::default();
    let mut buckets: Vec<SectionBucket> = Vec::new();
    let mut probe = CandidateProbe::of(contents, get_or_build_plan_of(language));
    let mut lines = get_lines_of(contents);
    let mut handed_back = None;
    while let Some((line_start, raw_line)) = handed_back.take().or_else(|| lines.next()) {
        let has_candidates = probe.has_a_candidate_in(line_start, line_start + raw_line.len());
        let had_code = walk_line::<EXPLAIN>(raw_line, line_start, language, collecting_spans,
                has_candidates, scan, &mut shell, &mut shell_stats, code_spans, log);

        // A region opener only counts where the shell left it as code, so one sitting inside a
        // comment or a string of the shell opens nothing
        if had_code && !language.nested_languages.is_empty()
                && let Some((region, inner)) = find_region_opening(raw_line.trim_ascii(), &scan.code_ranges, language, lookup) {
            let section_from = end_of_line(contents, line_start, raw_line);
            // A section is only a section if it closes. Nothing forces an opener to be a tag rather
            // than the same text written inside one, and on the strength of one word that is not a
            // tag the whole rest of the file would go to another language.
            let Some(closer_at) = find_tag_ignoring_case(&contents.as_bytes()[section_from..],
                    region.end.as_bytes()) else { continue };
            let closer_at = section_from + closer_at;

            // The tag line itself belongs to the shell, and anything it left open is cut off at
            // the section boundary: per the HTML reading, what follows the tag is section content
            shell = WalkState::default();

            let bucket_at = match buckets.iter().position(|bucket| bucket.language.name == inner.name) {
                Some(at) => at,
                None => {
                    buckets.push(SectionBucket { language: inner,
                    stats: if config.count_keywords { FileStats::with_keywords(&inner.keywords) }
                            else { FileStats::default() }, spans: Vec::new(),
                    collecting_spans: config.count_keywords && matchers.for_language(inner).is_some(),
                    bytes: 0 });
                    buckets.len() - 1
                }
            };
            let bucket = &mut buckets[bucket_at];
            let mut inner_state = WalkState::default();
            let mut section_to = contents.len();
            for (inner_start, inner_raw) in lines.by_ref() {
                // Per the HTML reading the closer ends the section wherever it stands, even inside
                // a string of the section's language: that is why one writes '<\/script>' in
                // JavaScript. The closer's line belongs to the shell.
                if inner_start + inner_raw.len() > closer_at {
                    section_to = inner_start;
                    handed_back = Some((inner_start, inner_raw));
                    break;
                }
                // A section is written in another language, whose symbols the probe never searched
                walk_line::<EXPLAIN>(inner_raw, inner_start, inner, bucket.collecting_spans,
                        true, scan, &mut inner_state, &mut bucket.stats, &mut bucket.spans, log);
            }
            bucket.bytes += section_to - section_from;
        }
    }

    if config.count_keywords {
        if let Some(matcher) = matchers.for_language(language) {
            count_keywords(contents, code_spans, matcher, &mut shell_stats, alias_indices);
        }
        for bucket in &mut buckets {
            if let Some(matcher) = matchers.for_language(bucket.language) {
                count_keywords(contents, &bucket.spans, matcher, &mut bucket.stats, alias_indices);
            }
        }
    }

    FileReport {
        shell: shell_stats,
        sections: buckets.into_iter().map(|bucket| SectionReport {
            language: bucket.language.name.clone(), stats: bucket.stats, bytes: bucket.bytes }).collect(),
        bytes: contents.len()
    }
}

// Returns whether the line left code behind, which is all the section machinery needs from it.
fn walk_line<const EXPLAIN: bool>(raw_line: &str, line_start: usize, language: &Language, collecting_spans: bool,
    has_candidates: bool, scan: &mut ScanBuffers, state: &mut WalkState, file_stats: &mut FileStats,
    code_spans: &mut Vec<(u32, u32)>, log: &mut ExplainLog) -> bool
{
    file_stats.lines += 1;
    let carried = if EXPLAIN { CarriedRecord::of(state) } else { CarriedRecord::Nothing };

    // Ascii-only trimming, since the unicode whitespace classification of trim() costs
    // a significant part of the total run time, for lines that are code either way
    let from_start = raw_line.trim_ascii_start();
    let line = from_start.trim_ascii_end();
    if line.is_empty() {
        // A line joined to a comment by a continuation symbol is inside that comment even when it
        // holds nothing, and the joining ends here: a blank line has nowhere to put the symbol
        // that would carry it further.
        let carried_by_a_continuation = state.continued_comment;
        state.continued_comment = false;
        let class = if state.open_str_symbol.is_some() { LineClass::BlankInString }
                else if state.open_comment.is_some() || carried_by_a_continuation { LineClass::BlankInComment }
                else { LineClass::Blank };
        file_stats.classes.bump(class);
        if EXPLAIN { log.record(class, carried, language, Vec::new()); }
        // A string whose symbol does not cross lines was only held open by a continuation, and
        // the continuation needs a backslash at the line's end, which a blank line has nowhere
        // to put. The line itself still counted inside the string, where it began.
        if state.open_str_symbol.is_some_and(|symbol| !language.string_crosses_lines(symbol)) {
            state.open_str_symbol = None;
        }
        return false;
    }
    let lead = raw_line.len() - from_start.len();
    let base = line_start + lead;

    // A line joined to the one before it by a continuation symbol is the tail of that line's
    // comment: in C '// a comment \' makes the whole next line comment too, however it is written.
    if state.continued_comment {
        let class = if has_word_byte(line.as_bytes()) { LineClass::WordsInComment }
                else { LineClass::PunctuationInComment };
        file_stats.classes.bump(class);
        state.continued_comment = ends_with_continuation(line, language);
        if EXPLAIN {
            log.record(class, carried, language,
                    vec![Span { from: lead, to: lead + line.len(), kind: SpanKind::Comment }]);
        }
        return false;
    }

    let mut line_spans: Vec<Span> = Vec::new();
    let (line_info, opened_here) = get_bounds::<EXPLAIN>(line, language, state.open_comment,
            state.open_str_symbol, has_candidates, scan, &mut line_spans);

    state.open_comment = line_info.open_comment_after;
    // Only a symbol declared to cross lines carries its string to the next one, so the damage of
    // an unbalanced quote is this line and not the rest of the file. A line ending in the
    // continuation symbol is the exception: there the language says the line goes on.
    state.open_str_symbol = line_info.open_str_symbol_after.filter(|symbol|
            language.string_crosses_lines(*symbol)
            || (continues_in(language, |continuation| continuation.in_strings)
                && ends_with_continuation(line, language)));

    // Each line lands in exactly one class, read off what it says and where it sits. The counting
    // models are folds over the classes at presentation time, so nothing here decides a bucket.
    let has_code = line_info.has_code;
    let words_in_code = has_code && has_word_byte_in(&scan.code_ranges, line);
    let counts_as_code = words_in_code || line_info.has_string_literal;
    let counts_as_comment = !counts_as_code && has_word_byte(line.as_bytes());

    // Whether this line ended inside a comment that the next one carries on. Only a line comment
    // reaches the continuation symbol: a closed block with a '\' after it, the shape of every C
    // macro holding a comment in its body, extends nothing, while 'code; // c \' joins the next
    // line to its comment the way C reads it. What the comment says is not asked, since a
    // decorative '// ---- \' continues as readily as a sentence. The symbol is tested last because
    // it is the most expensive question here and the one that is almost always no.
    state.continued_comment = state.open_str_symbol.is_none() && state.open_comment.is_none()
            && opened_here.ended_in_line_comment
            && continues_in(language, |continuation| continuation.in_comments)
            && ends_with_continuation(line, language);

    let class = if words_in_code { LineClass::WordsInCode }
            else if line_info.has_string_literal { LineClass::StringContent }
            else if counts_as_comment {
                if has_code { LineClass::CommentWordsBesideCode } else { LineClass::WordsInComment }
            }
            else if has_code { LineClass::PunctuationInCode }
            else { LineClass::PunctuationInComment };
    file_stats.classes.bump(class);
    if EXPLAIN {
        if (state.open_comment.is_some() && opened_here.comment)
                || (state.open_str_symbol.is_some() && opened_here.string)
                || state.continued_comment {
            state.opened_line = log.get_current_line_number();
        }
        for span in &mut line_spans {
            span.from += lead;
            span.to += lead;
        }
        log.record(class, carried.with_its_end_marked(state, opened_here), language, line_spans);
    }

    if counts_as_code && collecting_spans && has_code {
        push_trimmed_spans(code_spans, &scan.code_ranges, line, base);
    }
    has_code
}

// Past the line and past its own newline, whichever width the file wrote that newline with
fn end_of_line(contents: &str, line_start: usize, raw_line: &str) -> usize {
    let mut end = line_start + raw_line.len();
    if contents.as_bytes().get(end) == Some(&b'\r') { end += 1; }
    if contents.as_bytes().get(end) == Some(&b'\n') { end += 1; }
    end
}

// A region opener that survived the shell's own reading of the line, with the language its section
// is in. None when the tag does not close on this line, when its section also closes on this line,
// or when no language can be found for it: all three count as shell.
fn find_region_opening<'a>(line: &str, code_ranges: &[(usize, usize)], language: &'a Language,
    lookup: &'a NestedLanguageLookup) -> Option<(&'a NestedLanguage, &'a Language)>
{
    let bytes = line.as_bytes();
    for (from, to) in code_ranges {
        let mut cursor = *from;
        while let Some(offset) = memchr::memchr(b'<', &bytes[cursor..*to]) {
            let at = cursor + offset;
            cursor = at + 1;
            for region in &language.nested_languages {
                if !starts_with_ignoring_case(&bytes[at..], region.start.as_bytes()) {
                    continue;
                }
                let after_start = at + region.start.len();
                // Where the name of the tag ends, so that '<scriptures>' is a word in a page and
                // not the opener of a script block
                match bytes.get(after_start) {
                    Some(byte) if byte.is_ascii_whitespace() || *byte == b'>' => (),
                    _ => continue
                }
                // The tag has to close on its own line; split over two, the line stays shell
                let Some(tag_close) = memchr::memchr(b'>', &bytes[after_start..]) else { continue };
                // A section that opens and closes on one line stays shell whole, tags and all
                if find_tag_ignoring_case(&bytes[after_start + tag_close..], region.end.as_bytes()).is_some() {
                    continue;
                }
                let tag_text = &line[after_start..after_start + tag_close];
                let named = find_attribute_value(tag_text, "lang")
                        .or_else(|| find_attribute_value(tag_text, "type").map(strip_mime_family));
                let inner = named.and_then(|value| lookup.find_by_spelling(value))
                        .or_else(|| lookup.find_by_spelling(&region.default));
                if let Some(inner) = inner {
                    return Some((region, inner));
                }
            }
        }
    }
    None
}

// The value of one attribute inside a tag's text, with either quote or none: lang="ts", lang='ts'
// and lang=ts all answer ts. The name has to be preceded by whitespace so that 'slang=' is not
// 'lang=', and the '=' may carry spaces around it.
fn find_attribute_value<'a>(tag_text: &'a str, name: &str) -> Option<&'a str> {
    let bytes = tag_text.as_bytes();
    let mut cursor = 0;
    while let Some(offset) = find_case_insensitive(&bytes[cursor..], name.as_bytes()) {
        let at = cursor + offset;
        cursor = at + 1;
        if at != 0 && !bytes[at - 1].is_ascii_whitespace() {
            continue;
        }
        let rest = tag_text[at + name.len()..].trim_ascii_start();
        let Some(value) = rest.strip_prefix('=') else { continue };
        let value = value.trim_ascii_start();
        return Some(match value.as_bytes().first() {
            Some(&quote @ (b'"' | b'\'')) => value[1..].split(quote as char).next().unwrap_or(""),
            _ => value.split_ascii_whitespace().next().unwrap_or("")
        });
    }
    None
}

// 'type="text/typescript"' names its language after the slash, and a bare 'type="module"' has none
fn strip_mime_family(value: &str) -> &str {
    value.rsplit('/').next().unwrap_or(value)
}

// Asked at every '<' of every line of a markup file, which is why it is a comparison and not a
// search for a match at zero: a search that answers "no" has read the rest of the line first, and
// on one long line that is the whole line once per '<'.
fn starts_with_ignoring_case(haystack: &[u8], needle: &[u8]) -> bool {
    !needle.is_empty() && haystack.len() >= needle.len()
            && haystack[..needle.len()].eq_ignore_ascii_case(needle)
}

// Both cases of the needle's first byte are searched, so '</SCRIPT>' is found as readily as
// '</script>'.
fn find_tag_ignoring_case(haystack: &[u8], needle: &[u8]) -> Option<usize> {
    let name = needle.strip_suffix(b">").unwrap_or(needle);
    let first = *name.first()?;
    memchr::memchr2_iter(first.to_ascii_lowercase(), first.to_ascii_uppercase(), haystack)
            .find(|at| closes_a_tag(&haystack[*at..], name))
}

// The closing tag is read the way 'find_region_opening' reads the opening one: the name, then
// whitespace or '>'. What sits between the name and the '>' is not this counter's business, and a
// browser agrees: measured, '</script >' and '</script foo>' both close the element while
// '</scriptfoo>' is a different name and closes nothing. A tag that never reaches a '>' on the line
// it began is not a tag, which is the same line the opener draws.
fn closes_a_tag(rest: &[u8], name: &[u8]) -> bool {
    if !starts_with_ignoring_case(rest, name) {
        return false;
    }
    match rest.get(name.len()) {
        Some(b'>') => true,
        Some(byte) if byte.is_ascii_whitespace() => {
            let line_end = memchr::memchr(b'\n', rest).unwrap_or(rest.len());
            memchr::memchr(b'>', &rest[name.len()..line_end]).is_some()
        },
        _ => false
    }
}

fn find_case_insensitive(haystack: &[u8], needle: &[u8]) -> Option<usize> {
    if needle.is_empty() || haystack.len() < needle.len() {
        return None;
    }
    (0..=haystack.len() - needle.len())
            .find(|&at| haystack[at..at + needle.len()].eq_ignore_ascii_case(needle))
}

// 'has_code' is whether the line left any code in 'ScanBuffers::code_ranges': a line whose code is
// only whitespace left none, and counts as neither code nor comment.
// An open comment travels with its depth, which is 1 for every pair that does not nest and the
// count of unclosed openers for one that does.
#[derive(Debug, PartialEq)]
struct LineInfo {
    has_code: bool,
    has_string_literal: bool,
    open_comment_after: Option<(u8, u32)>,
    open_str_symbol_after: Option<u8>
}

impl LineInfo {
    fn of(has_code: bool, has_string_literal: bool) -> LineInfo {
        LineInfo { has_code, has_string_literal, open_comment_after: None, open_str_symbol_after: None }
    }

    fn with_open_comment(has_code: bool, has_string_literal: bool, symbol: u8, depth: u32) -> LineInfo {
        LineInfo { has_code, has_string_literal, open_comment_after: Some((symbol, depth)), open_str_symbol_after: None }
    }

    fn with_open_string(has_code: bool, symbol: Option<u8>) -> LineInfo {
        LineInfo { has_code, has_string_literal: true, open_comment_after: None, open_str_symbol_after: symbol }
    }
}

// What a character literal is allowed to hold: one character, or an escape sequence, which is what
// tells a real literal from two unrelated symbols that happen to sit on one line. Without it a
// lifetime's tick pairs with the apostrophe of a word inside a string, the false literal swallows
// that string's opening quote, and the quote left over carries to the end of the file. A single
// byte above ASCII cannot be judged alone, so anything that is one whole character passes.
fn holds_one_character(between: &[u8]) -> bool {
    match between.first() {
        None => false,
        Some(b'\\') => true,
        Some(byte) if byte.is_ascii() => between.len() == 1,
        // The leading byte of a multi-byte character, so the run has to be exactly that character
        Some(_) => std::str::from_utf8(between).is_ok_and(|text| text.chars().count() == 1)
    }
}

fn continues_in(language: &Language, wanted: impl Fn(&LineContinuation) -> bool) -> bool {
    language.line_continuation.as_ref().is_some_and(wanted)
}

// The symbol has to be the last thing on the line and not itself escaped, so a Windows path ending
// in a backslash inside a raw string does not join the next line to it.
fn ends_with_continuation(line: &str, language: &Language) -> bool {
    let Some(continuation) = &language.line_continuation else { return false };
    let bytes = line.as_bytes();
    bytes.ends_with(continuation.symbol.as_bytes())
            && is_not_escaped(bytes.len() - continuation.symbol.len(), bytes, language.strings.get_escape())
}

// A stretch of nothing but whitespace is not code, and recording one would say that the line has
// code on it: the space in '*/ /*' between a comment closing and the next one opening is the
// shape that reaches here.
fn push_code(ranges: &mut Vec<(usize, usize)>, line: &str, from: usize, to: usize) {
    if to > from && !line[from..to].trim_ascii().is_empty() {
        ranges.push((from, to));
    }
}

// A word byte is what makes a line say something: a letter, a digit, or anything above ASCII, so
// an identifier in a non-latin alphabet reads as content. Everything else is punctuation some
// grammar required: '}', '});', a bare '*/', the '*' of a banner line.
fn has_word_byte(bytes: &[u8]) -> bool {
    bytes.iter().any(|byte| byte.is_ascii_alphanumeric() || *byte >= 0x80)
}

fn has_word_byte_in(ranges: &[(usize, usize)], line: &str) -> bool {
    let bytes = line.as_bytes();
    ranges.iter().any(|(from, to)| has_word_byte(&bytes[*from..*to]))
}

// What reading a line learned that 'LineInfo' does not say. 'comment' and 'string' are whether the
// thing the line leaves open began on that same line, as opposed to carrying on from an earlier
// one; only '--explain' reads those, for the report of which line opened what.
// 'ended_in_line_comment' is read by the counting itself, to decide whether a continuation symbol
// at the end of the line carries the comment onto the next one.
#[derive(Debug, Default, Clone, Copy, PartialEq)]
struct OpenedHere {
    comment: bool,
    string: bool,
    ended_in_line_comment: bool,
}

// Only '--explain' collects spans; in the counting build this compiles to nothing at all, which
// was measured to matter: the same check done at run time cost a few percent of user CPU.
fn note_span<const EXPLAIN: bool>(spans: &mut Vec<Span>, from: usize, to: usize, kind: SpanKind) {
    if EXPLAIN && to > from {
        spans.push(Span { from, to, kind });
    }
}

fn get_bounds<const EXPLAIN: bool>(line: &str, language: &Language, open_comment: Option<(u8, u32)>,
    open_str_symbol: Option<u8>, has_candidates: bool, buffers: &mut ScanBuffers, spans: &mut Vec<Span>)
-> (LineInfo, OpenedHere)
{
    // A line holding none of the searched bytes cannot hold a symbol, so it lands where the scan
    // below lands when it finds nothing. The code ranges are rewritten rather than left as the line
    // before them left them, since the keyword search and the search for a nested language's
    // opening tag both read them off a line that came back as code.
    if !has_candidates {
        buffers.code_ranges.clear();
        if let Some((symbol, depth)) = open_comment {
            note_span::<EXPLAIN>(spans, 0, line.len(), SpanKind::Comment);
            return (LineInfo::with_open_comment(false, false, symbol, depth), OpenedHere::default());
        }
        if open_str_symbol.is_some() {
            note_span::<EXPLAIN>(spans, 0, line.len(), SpanKind::String);
            return (LineInfo::with_open_string(false, open_str_symbol), OpenedHere::default());
        }
        push_code(&mut buffers.code_ranges, line, 0, line.len());
        note_span::<EXPLAIN>(spans, 0, line.len(), SpanKind::Code);
        return (LineInfo::of(true, false), OpenedHere::default());
    }

    // A line comment runs to the end of its line, so a line that opens with one is comment through
    // and through and nothing the scan could find past it changes that. Only with nothing left open
    // above, since inside a block or a crossing string the same bytes are text.
    // The buffers are left as the line before them left them, which is safe only because nothing
    // reads them when no code span comes back.
    if open_comment.is_none() && open_str_symbol.is_none()
            && get_or_build_plan_of(language).line_comment_ends_the_line
            && language.comment_symbols.iter().any(|symbol| line.as_bytes().starts_with(symbol.as_bytes())
                    && stands_as_its_own_word(line.as_bytes(), 0, symbol.len())) {
        note_span::<EXPLAIN>(spans, 0, line.len(), SpanKind::Comment);
        return (LineInfo::of(false, false),
                OpenedHere { ended_in_line_comment: true, ..OpenedHere::default() });
    }

    scan_line(line, language, buffers);
    resolve_string_delimiters(language, open_str_symbol, buffers);
    let ScanBuffers { strings: str_indices, string_symbols: str_symbols, comments: comment_indices,
            com_starts: com_start_indices, com_ends: com_end_indices, code_ranges, .. } = buffers;

    match open_comment {
        None => if open_str_symbol.is_some() && str_indices.is_empty() {
            note_span::<EXPLAIN>(spans, 0, line.len(), SpanKind::String);
            return (LineInfo::with_open_string(false, open_str_symbol), OpenedHere::default());
        },
        // Only the end of the pair that opened the block closes it, so a line holding none of
        // those is comment through and through, whatever other symbols sit on it. A start of a
        // nesting pair counts as an event too, since it changes the depth.
        Some((open_pair, carried)) => {
            let leveled = language.comment_is_leveled(open_pair);
            let has_end = com_end_indices.iter().any(|(_, symbol, level)|
                    *symbol == open_pair && (!leveled || *level as u32 == carried));
            let deepens = language.comment_nests(open_pair)
                    && com_start_indices.iter().any(|(_, symbol, _)| *symbol == open_pair);
            if !has_end && !deepens {
                note_span::<EXPLAIN>(spans, 0, line.len(), SpanKind::Comment);
                return (LineInfo::with_open_comment(false, false, open_pair, carried), OpenedHere::default());
            }
        }
    }

    resolve_comment_and_multiline_end_overlap(line, language, comment_indices, com_end_indices);

    resolve_comment_and_multiline_start_overlap(line, language, comment_indices, com_start_indices);

    if !com_end_indices.is_empty() && !com_start_indices.is_empty() {
        resolve_double_counting_of_adjacent_start_and_end_symbols(com_start_indices, com_end_indices,
            open_comment.is_some(), language);
    }

    if str_indices.is_empty() && comment_indices.is_empty() && com_start_indices.is_empty() && com_end_indices.is_empty() {
        push_code(code_ranges, line, 0, line.len());
        note_span::<EXPLAIN>(spans, 0, line.len(), SpanKind::Code);
        return (LineInfo::of(true, false), OpenedHere::default());
    }

    let (mut start_com_counter, mut end_com_counter, mut str_counter, mut comment_counter) = (0,0,0,0);
    let (mut open_com_m, mut is_str_open_m) = (open_comment, open_str_symbol.is_some());
    let mut opened = OpenedHere::default();
    // Where the span being recorded for '--explain' began. Zero serves whichever of the three kinds
    // the line starts inside, and every transition below moves it past the span it just noted.
    let mut region_from = 0;

    let has_more_comments = |counter| counter < comment_indices.len(); 
    let has_more_strs = |counter| counter < str_indices.len();
    let has_more_ends = |counter| counter < com_end_indices.len();
    let has_more_starts = |counter| counter < com_start_indices.len();
    let next_symbol_is_comment = |comment_counter: usize, str_counter: usize,
        start_counter: usize| {
        if !has_more_comments(comment_counter) {return false; }
        if has_more_strs(str_counter) && comment_indices[comment_counter] > str_indices[str_counter] {
            return false;
        }
        if has_more_starts(start_counter) && comment_indices[comment_counter] > com_start_indices[start_counter].0 {
            return false;
        }
        true
    };
    let next_symbol_is_string = |comment_counter: usize, str_counter: usize,
        start_counter: usize| {
        if !has_more_strs(str_counter) {return false;}
        if has_more_comments(comment_counter)  && str_indices[str_counter] > comment_indices[comment_counter] {
            return false;
        }
        if has_more_starts(start_counter) && str_indices[str_counter] > com_start_indices[start_counter].0 {
            return false;
        }
        true
    };
    let next_symbol_is_com_start = |comment_counter: usize, str_counter: usize,
        start_counter: usize| {
        if !has_more_starts(start_counter) {return false;}
        if has_more_comments(comment_counter) && com_start_indices[start_counter].0 > comment_indices[comment_counter] {
            return false;
        }
        if has_more_strs(str_counter) && com_start_indices[start_counter].0 > str_indices[str_counter] {
            return false;
        }
        true
    };
    let progress_counters_after = |index, comment_counter: &mut usize, str_counter: &mut usize,
        start_counter: &mut usize, end_counter: &mut usize| {
        while *comment_counter < comment_indices.len() && comment_indices[*comment_counter] < index {
            *comment_counter += 1;
        }
        while *str_counter < str_indices.len() && str_indices[*str_counter] < index {
            *str_counter += 1;
        }
        while *start_counter < com_start_indices.len() && com_start_indices[*start_counter].0 < index {
            *start_counter += 1;
        }
        while *end_counter < com_end_indices.len() && com_end_indices[*end_counter].0 < index {
            *end_counter += 1;
        }
    };
    let skipped_com_end_symbol = |last_symbol_index: usize, end_com_counter: usize, cur_index: usize| {
        has_more_ends(end_com_counter) && com_end_indices[end_com_counter].0 < cur_index && com_end_indices[end_com_counter].0 >= last_symbol_index
    };

    let mut has_string_literal = false;
    let mut slice_start_index = 0;
    let mut last_symbol_index = 0;
    loop {
        if is_str_open_m {
            last_symbol_index = str_indices[str_counter];
            let index_after = last_symbol_index
                    + language.get_string_pair_of(str_symbols[str_counter]).1.len();
            if index_after >= line.len() {
                note_span::<EXPLAIN>(spans, region_from, line.len(), SpanKind::String);
                return (LineInfo::of(!code_ranges.is_empty(), true), OpenedHere::default());
            }
            note_span::<EXPLAIN>(spans, region_from, index_after, SpanKind::String);
            region_from = index_after;

            progress_counters_after(last_symbol_index, &mut comment_counter, &mut str_counter,
                    &mut start_com_counter, &mut end_com_counter);

            is_str_open_m = false;
            str_counter += 1;
            has_string_literal = true;
            slice_start_index = index_after;
        } else if let Some((open_pair, carried)) = open_com_m {
            // Ends of the other pairs inside this block are text. Walking the counters past them
            // is safe: everything before the closing position is dead once the block closes there.
            // For a pair that nests, each of its own starts before an end deepens the block, and
            // the closer is the end at which the count comes back to zero. For a leveled pair,
            // 'carried' is the level and only an end with the same count is looked at.
            let leveled = language.comment_is_leveled(open_pair);
            let nests = language.comment_nests(open_pair);
            let mut depth = if leveled { 1 } else { carried };
            let closing = loop {
                while end_com_counter < com_end_indices.len()
                        && (com_end_indices[end_com_counter].1 != open_pair
                            || (leveled && com_end_indices[end_com_counter].2 as u32 != carried)) {
                    end_com_counter += 1;
                }
                if end_com_counter == com_end_indices.len() { break None; }
                let end_at = com_end_indices[end_com_counter].0;

                if nests {
                    while start_com_counter < com_start_indices.len() && com_start_indices[start_com_counter].0 < end_at {
                        if com_start_indices[start_com_counter].1 == open_pair { depth = depth.saturating_add(1); }
                        start_com_counter += 1;
                    }
                }
                depth -= 1;
                if depth == 0 { break Some(end_at); }
                end_com_counter += 1;
            };
            let Some(closed_at) = closing else {
                let mut carry = carried;
                if nests {
                    while start_com_counter < com_start_indices.len() {
                        if com_start_indices[start_com_counter].1 == open_pair { depth = depth.saturating_add(1); }
                        start_com_counter += 1;
                    }
                    carry = depth;
                }
                note_span::<EXPLAIN>(spans, region_from, line.len(), SpanKind::Comment);
                return (LineInfo::with_open_comment(has_string_literal || !code_ranges.is_empty(),
                        has_string_literal, open_pair, carry), opened);
            };
            last_symbol_index = closed_at;
            let end_level = if leveled { carried as u8 } else { 0 };
            let index_after = last_symbol_index + language.comment_end_len(open_pair, end_level);
            if index_after >= line.len() {
                note_span::<EXPLAIN>(spans, region_from, line.len(), SpanKind::Comment);
                return (LineInfo::of(!code_ranges.is_empty(), has_string_literal), OpenedHere::default());
            }
            note_span::<EXPLAIN>(spans, region_from, index_after, SpanKind::Comment);
            region_from = index_after;

            // Every counter goes past the closer's own bytes and not merely past where it began, so
            // that a symbol standing at the byte after it is reached by the ordinary dispatch below
            // instead of being handled a second time here.
            open_com_m = None;
            progress_counters_after(index_after, &mut comment_counter, &mut str_counter,
                    &mut start_com_counter, &mut end_com_counter);
            slice_start_index = index_after;
        } else {
            if next_symbol_is_comment(comment_counter, str_counter, start_com_counter) {
                let comment_at = comment_indices[comment_counter];
                push_code(code_ranges, line, slice_start_index, comment_at);
                note_span::<EXPLAIN>(spans, region_from, comment_at, SpanKind::Code);
                note_span::<EXPLAIN>(spans, comment_at, line.len(), SpanKind::Comment);
                let ends = OpenedHere { ended_in_line_comment: true, ..OpenedHere::default() };
                return (LineInfo::of(!code_ranges.is_empty(), has_string_literal), ends);
            } else if next_symbol_is_string(comment_counter, str_counter, start_com_counter) {
                let this_index = str_indices[str_counter];
                if skipped_com_end_symbol(last_symbol_index, end_com_counter, this_index) {
                    end_com_counter += 1;
                }
                push_code(code_ranges, line, slice_start_index, this_index);
                note_span::<EXPLAIN>(spans, region_from, this_index, SpanKind::Code);
                region_from = this_index;
                str_counter += 1;
                if !has_more_strs(str_counter) {
                    note_span::<EXPLAIN>(spans, region_from, line.len(), SpanKind::String);
                    return (LineInfo::with_open_string(!code_ranges.is_empty(), Some(str_symbols[str_counter-1])),
                            OpenedHere { string: true, ..OpenedHere::default() });
                }

                is_str_open_m = true;
                has_string_literal = true;
                last_symbol_index = this_index;
            } else if next_symbol_is_com_start(comment_counter, str_counter, start_com_counter) {
                let (this_index, this_symbol, this_level) = com_start_indices[start_com_counter];
                if skipped_com_end_symbol(last_symbol_index, end_com_counter, this_index) {
                    end_com_counter += 1;
                }

                push_code(code_ranges, line, slice_start_index, this_index);
                note_span::<EXPLAIN>(spans, region_from, this_index, SpanKind::Code);
                region_from = this_index;
                // A nesting or leveled pair falls through to the open branch even with no ends
                // left: further starts of a nesting one still deepen the carried state, and the
                // leveled one carries its level either way
                if !has_more_ends(end_com_counter) && !language.comment_nests(this_symbol)
                        && !language.comment_is_leveled(this_symbol) {
                    note_span::<EXPLAIN>(spans, region_from, line.len(), SpanKind::Comment);
                    return (LineInfo::with_open_comment(has_string_literal || !code_ranges.is_empty(),
                            has_string_literal, this_symbol, 1), OpenedHere { comment: true, ..OpenedHere::default() });
                }

                open_com_m = Some((this_symbol,
                        if language.comment_is_leveled(this_symbol) { this_level as u32 } else { 1 }));
                opened.comment = true;
                start_com_counter += 1;
                last_symbol_index = this_index;
            } else {
                push_code(code_ranges, line, slice_start_index, line.len());
                note_span::<EXPLAIN>(spans, region_from, line.len(), SpanKind::Code);
                return (LineInfo::of(true, has_string_literal), OpenedHere::default());
            }
        }
    }
}

// The collision window around each symbol is that symbol's own span: an end beginning inside a
// start's bytes, or a start beginning inside an end's bytes. One shared length for both sides is
// wrong wherever a pair's halves differ in length, as Lua's and HTML's do: ']]--[[' would read as
// a collision when the two symbols merely touch, and the reopening start would be discarded.
fn resolve_double_counting_of_adjacent_start_and_end_symbols(start_indices: &mut Vec<(usize, u8, u8)>,
    end_indices: &mut Vec<(usize, u8, u8)>, is_comment_open: bool, language: &Language)
{
    fn resolve_collision(start_indices: &mut Vec<(usize, u8, u8)>, end_indices: &mut Vec<(usize, u8, u8)>, start_counter: &mut usize,
        end_counter: &mut usize, is_comment_open_m: &mut bool, language: &Language)
    {
        if *is_comment_open_m {
            start_indices.remove(*start_counter);
            if *start_counter < start_indices.len() && start_indices[*start_counter].0 <
                    end_indices[*end_counter].0 + language.comment_end_len(end_indices[*end_counter].1, end_indices[*end_counter].2) {
                start_indices.remove(*start_counter);
            }
            *end_counter += 1;
        } else {
            end_indices.remove(*end_counter);
            if *end_counter < end_indices.len() && end_indices[*end_counter].0 <
                    start_indices[*start_counter].0 + language.comment_start_len(start_indices[*start_counter].1, start_indices[*start_counter].2) {
                end_indices.remove(*end_counter);
            }
            *start_counter += 1;
        }
        *is_comment_open_m = !*is_comment_open_m;
    }

    let mut is_comment_open_m = is_comment_open;
    let (mut start_counter, mut end_counter) = (0,0);
    loop {
        if start_counter == start_indices.len() || end_counter == end_indices.len() {break;}

        let (start_index, start_symbol, start_level) = start_indices[start_counter];
        let (end_index, end_symbol, end_level) = end_indices[end_counter];

        if end_index > start_index && end_index < start_index + language.comment_start_len(start_symbol, start_level) ||
                start_index > end_index && start_index < end_index + language.comment_end_len(end_symbol, end_level) {
            resolve_collision(start_indices, end_indices, &mut start_counter, &mut end_counter, &mut is_comment_open_m, language);
        } else {
            if start_index < end_index {
                start_counter += 1;
                if start_counter < start_indices.len() {
                    if start_indices[start_counter].0 > end_index {
                        is_comment_open_m = true;
                    }
                } else {
                    break;
                }
            }
            else {
                end_counter += 1;
                if end_counter < end_indices.len() {
                    if end_indices[end_counter].0 > start_counter {
                        is_comment_open_m = false;
                    }
                } else {
                    break;
                }
            }
        }
    }
}

// The trim decides whether a keyword at the start of the line has an acceptable prefix: a tab is
// not one, an empty prefix is. Only the front of the first stretch and the back of the last are
// trimmed, and a stretch that empties out completely is dropped.
fn push_trimmed_spans(spans: &mut Vec<(u32, u32)>, ranges: &[(usize, usize)], line: &str, base: usize) {
    let bytes = line.as_bytes();
    let (mut head, mut tail) = (0usize, ranges.len());
    let (mut head_from, mut tail_to) = (0usize, 0usize);

    while head < tail {
        let (from, to) = ranges[head];
        let mut at = from;
        while at < to && bytes[at].is_ascii_whitespace() { at += 1; }
        if at < to { head_from = at; break; }
        head += 1;
    }
    if head == tail { return; }

    while tail > head {
        let (from, to) = ranges[tail - 1];
        let floor = if tail - 1 == head { head_from } else { from };
        let mut at = to;
        while at > floor && bytes[at - 1].is_ascii_whitespace() { at -= 1; }
        if at > floor { tail_to = at; break; }
        tail -= 1;
    }

    for (i, (from, to)) in ranges.iter().enumerate().take(tail).skip(head) {
        let from = if i == head { head_from } else { *from };
        let to = if i == tail - 1 { tail_to } else { *to };
        spans.push(((base + from) as u32, (base + to) as u32));
    }
}

// A hit counts only if it lies entirely inside one stretch of code, and its neighbours are read
// inside that same stretch, so what a string literal removed is not treated as touching what
// follows it.
fn count_keywords(contents: &str, spans: &[(u32, u32)], matcher: &KeywordMatcher,
    file_stats: &mut FileStats, indices: &mut Vec<usize>)
{
    // The two sides are different questions and '(' is where they part. After the word it opens an
    // argument list and belongs to the declaration: Delphi's 'TFoo = class(TObject)' and Erlang's
    // '-module(greeter).' count as nothing at all if it is refused. Before the word it means the word
    // heads an s-expression, which the alias already handles by including the bracket, as Clojure
    // does with '(defn'; accepting it there too counts '(defn' twice, once through each alias.
    fn is_acceptable_before(byte: Option<&u8>) -> bool {
        match byte {
            None => true,
            Some(b) => *b == b' ' || *b == b'}' || *b == b'{' || *b == b','
        }
    }

    fn is_acceptable_after(byte: Option<&u8>) -> bool {
        matches!(byte, Some(b'(')) || is_acceptable_before(byte)
    }

    if spans.is_empty() { return; }
    let bytes = contents.as_bytes();

    for (alias_finder, alias_len, keyword_index) in &matcher.aliases_with_indices {
        indices.clear();
        indices.extend(alias_finder.find_iter(bytes));
        if indices.is_empty() { continue; }

        // both lists ascend, so the stretch that could hold the next hit is never behind us
        let mut span = 0;
        for (found, at) in indices.iter().enumerate() {
            // A hit touching another of the same alias is part of a longer word and never counts
            if (found > 0 && indices[found - 1] + alias_len == *at)
                    || indices.get(found + 1).is_some_and(|next| at + alias_len == *next) {
                continue;
            }

            while span < spans.len() && (spans[span].1 as usize) <= *at { span += 1; }
            if span == spans.len() { break; }

            let (from, to) = (spans[span].0 as usize, spans[span].1 as usize);
            if *at < from || at + alias_len > to { continue; }

            let before = if *at > from { bytes.get(*at - 1) } else { None };
            let after = if at + alias_len < to { bytes.get(at + alias_len) } else { None };
            if is_acceptable_before(before) && is_acceptable_after(after) {
                file_stats.keyword_occurences[*keyword_index] += 1;
            }
        }
    }
}

// Every string symbol the scan found, reduced to the ones that actually open or close a string.
// Only the symbol that opened a string can close it, so anything of another kind in between is
// text. A pair whose halves differ splits the rule in two: its opener cannot close and its closer
// cannot open, so a stray '"#' sitting in code is text and not the start of anything.
fn resolve_string_delimiters(language: &Language, open_str_symbol: Option<u8>, buffers: &mut ScanBuffers) {
    let ScanBuffers { raw_strings, strings, string_symbols, .. } = buffers;

    let mut open = open_str_symbol;
    let mut consumed_up_to = 0;

    for &(at, symbol, role) in raw_strings.iter() {
        // What sits inside a symbol that was already taken is part of it, not a symbol of its own
        if at < consumed_up_to {
            continue;
        }
        let length = match open {
            Some(open_symbol) => {
                if open_symbol != symbol || role == ROLE_OPEN { continue; }
                open = None;
                language.get_string_pair_of(symbol).1.len()
            }
            None => {
                // A closer opens nothing, and neither does a raw symbol the language escaped:
                // outside a string there is nothing for the escape to be an ordinary byte of
                if role == ROLE_CLOSE || role == ROLE_RAW_ESCAPED { continue; }
                open = Some(symbol);
                language.get_string_pair_of(symbol).0.len()
            }
        };
        consumed_up_to = at + length;
        strings.push(at);
        string_symbols.push(symbol);
    }
}

// When a comment symbol and a multiline start overlap only one of them is real: whichever begins
// first swallows the other, and on a tie the longer one wins. All three shapes occur. A '/*' inside a
// '//' opens nothing. PowerShell's '<#' contains a '#', and reading that as a comment of its own
// stops the block ever opening, which silently breaks every block comment in the language. Lua's
// '--[[' begins exactly where its own '--' does, with the same result if the shorter one wins.
fn resolve_comment_and_multiline_start_overlap(line: &str, language: &Language,
    comment_indices: &mut Vec<usize>, com_start_indices: &mut Vec<(usize, u8, u8)>)
{
    if comment_indices.is_empty() || com_start_indices.is_empty() {
        return;
    }
    let longest_comment_at = |at: usize| {
        language.comment_symbols.iter()
                .filter(|symbol| line.as_bytes()[at..].starts_with(symbol.as_bytes()))
                .map(String::len)
                .max()
                .unwrap_or(0)
    };

    com_start_indices.retain(|(start, _, _)| !comment_indices.iter()
            .any(|at| start > at && *start < at + longest_comment_at(*at)));
    comment_indices.retain(|at| !com_start_indices.iter()
            .any(|(start, symbol, level)| at > start && *at < start + language.comment_start_len(*symbol, *level)));

    // On a tie the longer symbol wins, and with several pairs the start at that position is the
    // longest of them, since same-position candidates arrive longest first
    comment_indices.retain(|at| match com_start_indices.iter().find(|(start, _, _)| start == at) {
        Some((_, symbol, level)) => longest_comment_at(*at) >= language.comment_start_len(*symbol, *level),
        None => true
    });
    com_start_indices.retain(|(at, _, _)| !comment_indices.contains(at));
}

// A comment symbol beginning inside a multiline end symbol is part of it and not a comment of its
// own. It is moved past that end symbol rather than dropped: no symbol is searched overlapping
// itself, so in '*///' the real '//' was already suppressed by the one lying across the closer,
// and discarding that one without giving its bytes back leaves the line with no comment at all.
fn resolve_comment_and_multiline_end_overlap(line: &str, language: &Language,
    comment_indices: &mut Vec<usize>, com_end_indices: &[(usize, u8, u8)])
{
    if comment_indices.is_empty() || com_end_indices.is_empty() {
        return;
    }
    let past_the_end_symbol_at = |at: usize| com_end_indices.iter().find_map(|(end, symbol, level)| {
        let after = end + language.comment_end_len(*symbol, *level);
        (at > *end && at < after).then_some(after)
    });
    let starts_a_comment = |at: usize| language.comment_symbols.iter()
            .any(|symbol| line.as_bytes()[at..].starts_with(symbol.as_bytes())
                    && stands_as_its_own_word(line.as_bytes(), at, symbol.len()));

    for at in comment_indices.iter_mut() {
        if let Some(after) = past_the_end_symbol_at(*at) {
            *at = after;
        }
    }
    comment_indices.retain(|at| *at < line.len() && starts_a_comment(*at));
    comment_indices.dedup();
}

#[cfg(test)]
mod tests {
    use std::sync::{Arc, LazyLock};

    use super::*;
    use crate::{CountingModel, Keyword, LineClasses, Stats, StringRules};
    use crate::test_paths::{FIXTURES_DIR, LANGUAGES_DIR};
    use crate::engine::identity::{ClaimKind, LanguageLookup, build_language_map_by};

    // The sample files carry no telling extension, because the language is the one the test names
    // and not the one a suffix would imply: that is what lets one file count as Java and as C#.
    fn sample_file(name: &str) -> std::path::PathBuf {
        Path::new(FIXTURES_DIR).join("parser").join(name)
    }

    // The parser hands back ranges into the line, and the text a test wants to read is rebuilt
    // from them here.
    #[derive(Debug, PartialEq)]
    struct TextInfo {
        cleansed_string: Option<String>,
        has_string_literal: bool,
        open_comment_after: Option<(u8, u32)>,
        open_str_symbol_after: Option<u8>
    }

    impl TextInfo {
        fn from_slice(slice: &str) -> TextInfo {
            TextInfo { cleansed_string: Some(slice.to_owned()), has_string_literal: false, open_comment_after: None, open_str_symbol_after: None }
        }
        fn from_slice_w_literal(slice: &str) -> TextInfo {
            TextInfo { cleansed_string: Some(slice.to_owned()), has_string_literal: true, open_comment_after: None, open_str_symbol_after: None }
        }
        fn with_open_comment(symbol: u8) -> TextInfo {
            TextInfo { cleansed_string: None, has_string_literal: false, open_comment_after: Some((symbol, 1)), open_str_symbol_after: None }
        }
        fn with_open_comment_at(symbol: u8, depth: u32) -> TextInfo {
            TextInfo { cleansed_string: None, has_string_literal: false, open_comment_after: Some((symbol, depth)), open_str_symbol_after: None }
        }
        fn with_open_symbol(symbol: u8) -> TextInfo {
            TextInfo { cleansed_string: None, has_string_literal: true, open_comment_after: None, open_str_symbol_after: Some(symbol) }
        }
        fn none_all(has_string_literal: bool) -> TextInfo {
            TextInfo { cleansed_string: None, has_string_literal, open_comment_after: None, open_str_symbol_after: None }
        }
        fn new(cleansed_string: Option<String>, has_string_literal: bool, open_comment_after: Option<(u8, u32)>, open_str_symbol_after: Option<u8>) -> TextInfo {
            TextInfo { cleansed_string, has_string_literal, open_comment_after, open_str_symbol_after }
        }
    }

    fn text_of(line: &str, info: LineInfo, buffers: &ScanBuffers) -> TextInfo {
        TextInfo {
            cleansed_string: info.has_code.then(||
                    buffers.code_ranges.iter().map(|(a, b)| &line[*a..*b]).collect::<String>()),
            has_string_literal: info.has_string_literal,
            open_comment_after: info.open_comment_after,
            open_str_symbol_after: info.open_str_symbol_after
        }
    }

    fn bounds_multi(line: &str, language: &Language, open_comment: Option<u8>, open_str_symbol: Option<u8>) -> TextInfo {
        bounds_multi_deep(line, language, open_comment.map(|symbol| (symbol, 1)), open_str_symbol)
    }

    fn bounds_multi_deep(line: &str, language: &Language, open_comment: Option<(u8, u32)>, open_str_symbol: Option<u8>) -> TextInfo {
        let mut buffers = ScanBuffers::default();
        let (info, _) = get_bounds::<false>(line, language, open_comment, open_str_symbol, true,
                &mut buffers, &mut Vec::new());
        text_of(line, info, &buffers)
    }

    fn keywords_of(line: &str, matcher: &KeywordMatcher, file_stats: &mut FileStats) {
        count_keywords(line, &[(0, line.len() as u32)], matcher, file_stats, &mut Vec::new());
    }

    fn str_delimiters(line: &str, language: &Language, open_str_symbol: Option<u8>) -> (Vec<usize>, Vec<u8>) {
        let mut buffers = ScanBuffers::default();
        scan_line(line, language, &mut buffers);
        resolve_string_delimiters(language, open_str_symbol, &mut buffers);
        (buffers.strings, buffers.string_symbols)
    }

    fn comment_delimiters(line: &str, language: &Language) -> Vec<usize> {
        let mut buffers = ScanBuffers::default();
        scan_line(line, language, &mut buffers);
        buffers.comments
    }

    fn comment_delimiters_w_multiline(line: &str, language: &Language, com_end_indices: &[usize]) -> Vec<usize> {
        let ends = com_end_indices.iter().map(|at| (*at, 0u8, 0u8)).collect::<Vec<_>>();
        let mut buffers = ScanBuffers::default();
        scan_line(line, language, &mut buffers);
        resolve_comment_and_multiline_end_overlap(line, language, &mut buffers.comments, &ends);
        buffers.comments
    }

    static CLASS : LazyLock<Keyword> = LazyLock::new(|| Keyword::new("classes", ["class"]));

    static INTERFACE : LazyLock<Keyword> = LazyLock::new(|| Keyword::new("interfaces", ["interface"]));

    static ENUM : LazyLock<Keyword> = LazyLock::new(|| Keyword::new("enums", ["enum"]));

    static STRUCT : LazyLock<Keyword> = LazyLock::new(|| Keyword::new("structs", ["struct"]));

    static TRAIT : LazyLock<Keyword> = LazyLock::new(|| Keyword::new("traits", ["trait"]));

    // The declaration behind most of the languages below: one double quote, cancelled by a
    // backslash, which is what the C family and everything shaped like it writes.
    fn build_backslashed_quotes() -> StringRules {
        StringRules::escaping_with(b'\\').with_symbols(["\""])
    }

    static JAVA : LazyLock<Language> = LazyLock::new(|| Language::new("java", ["java"],
            build_backslashed_quotes(), ["//"], &[("/*", "*/")], [CLASS.clone(), INTERFACE.clone()]));

    static PHP : LazyLock<Language> = LazyLock::new(|| Language::new("PHP", ["php"],
            StringRules::escaping_with(b'\\').with_symbols(["\"", "'"]), ["//", "#"],
            &[("/*", "*/")], [CLASS.clone()]));

    static PYTHON : LazyLock<Language> = LazyLock::new(|| Language::new("py", ["py"],
            StringRules::escaping_with(b'\\').with_symbols(["\"", "'"]), ["#"], &[], [CLASS.clone()]));

    static RUST : LazyLock<Language> = LazyLock::new(|| Language::new("rust", ["rs"],
            build_backslashed_quotes(), ["//"], &[("/*", "*/")],
            [STRUCT.clone(), ENUM.clone(), TRAIT.clone()]));

    // Four string symbols and three comment ones, past the two of each that most languages declare.
    // The docstring symbols sit among the ones that cross lines, which numbers them after the
    // plain quotes.
    static PYTHON_FULL : LazyLock<Language> = LazyLock::new(|| Language::new("py", ["py"],
            StringRules::escaping_with(b'\\').with_symbols(["\"", "'"])
                    .with_multiline_strings(["\"\"\"", "'''"]),
            ["#", "//", "--"], &[], [CLASS.clone()]));

    static LANGUAGE_MAP_REF : LazyLock<Arc<HashMap<String,Language>>> = LazyLock::new(||
            Arc::new(crate::languages::keyed_by_name(crate::language_file::parse_languages_in_dir(LANGUAGES_DIR).unwrap().0)));

    static JAVA_MATCHER : LazyLock<KeywordMatcher> = LazyLock::new(|| KeywordMatcher::build(&JAVA).unwrap());

    static NO_EXTENSIONS : LazyLock<HashMap<String, Arc<str>>> = LazyLock::new(HashMap::new);
    static NO_SET_ASIDE : LazyLock<HashMap<String, Language>> = LazyLock::new(HashMap::new);

    // With the real extension map, so a fixture or a stress case whose sections name a language
    // resolves it the way a run does; without priority rules, which no fixture contests
    static SHIPPED_EXTENSIONS : LazyLock<HashMap<String, Arc<str>>> = LazyLock::new(||
            build_language_map_by(ClaimKind::Extension, &LANGUAGE_MAP_REF, &HashMap::new(), &HashMap::new()).0);

    fn shipped_lookup() -> NestedLanguageLookup<'static> {
        NestedLanguageLookup { languages: &LANGUAGE_MAP_REF, extension_to_name: &SHIPPED_EXTENSIONS, set_aside: &NO_SET_ASIDE }
    }

    fn parse_file_whole(path: &Path, lang_name: &str, buf: &mut Vec<u8>, config: &EngineConfig) -> Result<FileStats, String> {
        parse_file_report(path, lang_name, buf, config).map(FileReport::into_whole)
    }

    fn parse_file_report(path: &Path, lang_name: &str, buf: &mut Vec<u8>, config: &EngineConfig) -> Result<FileReport, String> {
        match parse_file(path, get_size_of(path), lang_name, buf, &mut ParseBuffers::default(), &shipped_lookup(),
                &mut KeywordMatchers::default(), &mut IdentificationMatchers::default(), config,
                false, None, &HashMap::new())? {
            FileOutcome::Counted(report, _) => Ok(report),
            FileOutcome::Skipped(kind) => panic!("{} was skipped as {}", path.display(), kind.name())
        }
    }

    // The size the directory scan would have handed the counting thread
    fn get_size_of(path: &Path) -> u64 {
        std::fs::metadata(path).map_or(0, |m| m.len())
    }

    fn parse_lines_whole(contents: &str, language: &Language) -> FileStats {
        parse_lines::<false>(contents, language, &NestedLanguageLookup { languages: &NO_SET_ASIDE,
                extension_to_name: &NO_EXTENSIONS, set_aside: &NO_SET_ASIDE },
                &mut KeywordMatchers::default(), &EngineConfig::default(), &mut ParseBuffers::default(),
                &mut ExplainLog::default()).into_whole()
    }

    fn c_like_with_a_splice() -> Language {
        Language::new("c-like", ["c"], build_backslashed_quotes(), ["//"], &[("/*", "*/")], [])
                .with_line_continuation("\\", true, true)
    }

    // 'int a = 1; // comment \' joins the next line to its comment in C, code on the line or not.
    // The closed block's '/* c */ \' is the shape that must never carry: there the line ends
    // outside every comment and the backslash is code, which is what every C macro body relies on.
    #[test]
    fn a_spliced_line_comment_carries_even_off_a_line_that_also_holds_code() {
        let language = c_like_with_a_splice();

        let stats = parse_lines_whole("int a = 1; // comment \\\n   joined to the comment\nint x = 1;\n", &language);
        assert_eq!(2, stats.classes.words_in_code);
        assert_eq!(1, stats.classes.words_in_comment);

        let stats = parse_lines_whole("/* block */ \\\nint x = 1;\n", &language);
        assert_eq!(1, stats.classes.words_in_code);
        assert_eq!(1, stats.classes.comment_words_beside_code);
    }

    // The line the splice joined holds nothing, and it is still that comment's line: under the
    // model that asks which block a line sits in it is a comment, and under the one that asks what
    // a line says it is as empty as any other blank, which is why the class and not a column says
    // so. The splice itself ends there, having nowhere to put the symbol that would carry it on.
    #[test]
    fn a_blank_line_the_splice_joined_to_a_comment_belongs_to_that_comment() {
        let language = c_like_with_a_splice();
        let stats = parse_lines_whole("// a comment \\\n\nint x = 1;\n", &language);
        assert_eq!(1, stats.classes.blank_in_comment);
        assert_eq!(0, stats.classes.blank);
        assert_eq!(1, stats.classes.words_in_comment);
        assert_eq!(1, stats.classes.words_in_code);

        // Two of them: the first ends the joining, so the second is an ordinary blank
        let stats = parse_lines_whole("// a comment \\\n\n\nint x = 1;\n", &language);
        assert_eq!(1, stats.classes.blank_in_comment);
        assert_eq!(1, stats.classes.blank);

        // A language that does not join lines at all keeps its blank
        let plain = Language::new("c-like", ["c"], build_backslashed_quotes(), ["//"], &[("/*", "*/")], []);
        let stats = parse_lines_whole("// a comment \\\n\nint x = 1;\n", &plain);
        assert_eq!(0, stats.classes.blank_in_comment);
        assert_eq!(1, stats.classes.blank);
    }

    // A symbol that crosses lines on its own is untouched by this, which is what keeps docstrings
    // whole.
    #[test]
    fn a_blank_line_ends_a_string_the_splice_was_carrying() {
        let stats = parse_lines_whole("char *s = \"abc \\\n\n// a comment\";\nint x = 1;\n",
                &c_like_with_a_splice());
        assert_eq!(2, stats.classes.words_in_code);
        assert_eq!(1, stats.classes.blank_in_string);
        assert_eq!(1, stats.classes.words_in_comment);

        let crossing = Language::new("py-like", ["py"],
                build_backslashed_quotes().with_multiline_strings(["\"\"\""]), ["#"], &[], []);
        let stats = parse_lines_whole("\"\"\" open\n\nstill inside \"\"\"\n", &crossing);
        assert_eq!(1, stats.classes.blank_in_string);
        assert_eq!(2, stats.classes.string_content);
    }

    // Seeded from the language and then given the one file, the way a real run does it: the seed
    // puts a slot in for every keyword the language declares, so one that never occurs still
    // reports its zero instead of being missing.
    fn content_info_of(file: FileStats, lang_name: &str) -> Stats {
        let language = LANGUAGE_MAP_REF.get(lang_name).unwrap();
        let mut stats = Stats::from(language);
        stats.add_file(&file, 0, &language.keywords);
        stats
    }

    // The three columns as the default view folds them
    fn content_counts(stats: &FileStats) -> (usize, usize, usize) {
        (stats.lines, CountingModel::Content.calculate_code_lines(&stats.classes),
                CountingModel::Content.calculate_comment_lines(&stats.classes))
    }

    #[test]
    fn a_sample_file_is_counted_under_each_language_that_claims_it_with_that_languages_keywords() {
        let mut buf = Vec::with_capacity(150);

        let assert_sample = |file: &str, lang: &str, counts: (usize, usize, usize),
                keywords: HashMap<String, usize>, config: &EngineConfig, buf: &mut Vec<u8>| {
            let file_stats = parse_file_whole(&sample_file(file), lang, buf, config).unwrap();
            assert_eq!(counts, content_counts(&file_stats), "{file} as {lang}");
            assert_eq!(keywords, content_info_of(file_stats, lang).keyword_occurences, "{file} as {lang}");
            buf.clear();
        };

        let mut config = EngineConfig::default();
        assert_sample("a.txt", "Java", (44, 13, 8),
                hashmap!("classes".to_owned()=>3,"interfaces".to_owned()=>0), &config, &mut buf);
        // The keywords keep their slots and stay at zero, which is what a run produces: the seed
        // comes from the language and not from the file, so hiding them stops the counting and not
        // the language's own list of what it would have counted.
        config.count_keywords = false;
        assert_sample("a.txt", "Java", (44, 13, 8),
                hashmap!("classes".to_owned()=>0,"interfaces".to_owned()=>0), &config, &mut buf);
        config.count_keywords = true;
        assert_sample("a.txt", "C#", (44, 13, 8),
                hashmap!("structs".to_owned()=>0,"classes".to_owned()=>3,"interfaces".to_owned()=>0), &config, &mut buf);

        assert_sample("d.txt", "C#", (19, 7, 7),
                hashmap!("structs".to_owned()=>0,"classes".to_owned()=>5,"interfaces".to_owned()=>0), &config, &mut buf);
        assert_sample("d.txt", "Java", (19, 7, 7),
                hashmap!("classes".to_owned()=>5,"interfaces".to_owned()=>0), &config, &mut buf);

        assert_sample("b.txt", "Java", (19, 11, 4),
                hashmap!("classes".to_owned()=>7,"interfaces".to_owned()=>0), &config, &mut buf);

        // The 'class' on the line between two lone apostrophes counts: Python declares its plain
        // quotes single-line, so the quote above it dies at its own line instead of swallowing it
        assert_sample("c.txt", "Python", (11, 6, 1),
                hashmap!("classes".to_owned()=>3), &config, &mut buf);
    }

    // The expected classes are a hand count of a.txt, so this is also the worked example of what
    // each class means.
    #[test]
    fn one_parse_answers_both_models_through_the_classes() {
        let mut buf = Vec::with_capacity(150);
        let stats = parse_file_whole(&sample_file("a.txt"), "Java", &mut buf, &EngineConfig::default()).unwrap();

        assert_eq!(LineClasses {
            words_in_code: 13, string_content: 0, comment_words_beside_code: 0, words_in_comment: 8,
            punctuation_in_code: 10, punctuation_in_comment: 7, blank: 6, blank_in_comment: 0,
            blank_in_string: 0
        }, stats.classes);

        assert_eq!((44, 13, 8), content_counts(&stats));
        assert_eq!(23, CountingModel::Region.calculate_code_lines(&stats.classes));
        assert_eq!(15, CountingModel::Region.calculate_comment_lines(&stats.classes));
    }

    // The buffers are seeded from a line that did hold code, because a fresh one is empty already
    // and would pass whether or not the shortcut clears what the line before it left behind.
    #[test]
    fn a_line_with_no_symbol_byte_on_it_reads_the_same_with_the_scan_skipped() {
        let line = "let total = width + height";
        assert!(!line.contains(['/', '*', '"', '\'']), "the line carries a symbol byte");

        let read = |has_candidates: bool, open_comment, open_str_symbol| {
            let mut buffers = ScanBuffers::default();
            get_bounds::<true>("let seeded = 1", &RUST, None, None, true, &mut buffers, &mut Vec::new());
            assert!(!buffers.code_ranges.is_empty(), "the seeding line left no code range behind");
            let mut spans = Vec::new();
            let answer = get_bounds::<true>(line, &RUST, open_comment, open_str_symbol, has_candidates,
                    &mut buffers, &mut spans);
            (answer, buffers.code_ranges.clone(), spans)
        };

        for (open_comment, open_str_symbol) in [(None, None), (Some((0u8, 1u32)), None), (None, Some(0u8))] {
            assert_eq!(read(true, open_comment, open_str_symbol), read(false, open_comment, open_str_symbol),
                    "the shortcut disagreed with the scan for {open_comment:?} and {open_str_symbol:?}");
        }
    }

    // A line is counted where it says something: words in code make it code, words only in a
    // comment make it a comment, and a line with no word anywhere is extra, because a bare
    // delimiter is the grammar's and not the writer's.
    #[test]
    fn a_line_counts_where_its_words_are_and_bare_delimiters_are_extra() {
        let counts = |contents: &str| content_counts(&parse_lines_whole(contents, &JAVA));

        // the banner shape: only the starred text is a comment, the ceremony around it is extra
        assert_eq!((4, 0, 1), counts("/*\n* words here\n*\n*/\n"));
        // a decorative separator says nothing
        assert_eq!((2, 1, 0), counts("/*----------*/\nint x = 1;\n"));
        // a bare line comment is a spacer, one with words is a comment
        assert_eq!((3, 1, 1), counts("//\n// words\nint x = 1;\n"));
        // the words of a mixed line sit in its comment, and the brace does not demote it
        assert_eq!((2, 1, 1), counts("int x = 1;\n} // end of main\n"));
        // a bare closer with a brace beside it says no more than the bare closer alone
        assert_eq!((3, 0, 2), counts("/* words\n*/ }\n/* more words\n"));
    }

    // A keyword cut in half by a string literal must not count: each surviving stretch is searched
    // where it lies rather than glued to the next one.
    #[test]
    fn a_keyword_split_by_a_string_is_not_a_keyword() {
        let line = "str\"X\"uct a;";
        let mut file_stats = FileStats::with_keywords(&[STRUCT.clone(),ENUM.clone(),TRAIT.clone()]);
        let matcher = KeywordMatcher::build(&RUST).unwrap();
        let mut buffers = ScanBuffers::default();
        let (info, _) = get_bounds::<false>(line, &RUST, None, None, true, &mut buffers, &mut Vec::new());
        let mut spans = Vec::new();
        assert!(info.has_code);
        push_trimmed_spans(&mut spans, &buffers.code_ranges, line, 0);
        count_keywords(line, &spans, &matcher, &mut file_stats, &mut Vec::new());
        assert_eq!(0, file_stats.keyword_occurences[0]);

        // and the same word, whole, still counts
        let line = "struct a;";
        let mut file_stats = FileStats::with_keywords(&[STRUCT.clone(),ENUM.clone(),TRAIT.clone()]);
        let mut buffers = ScanBuffers::default();
        let (info, _) = get_bounds::<false>(line, &RUST, None, None, true, &mut buffers, &mut Vec::new());
        let mut spans = Vec::new();
        assert!(info.has_code);
        push_trimmed_spans(&mut spans, &buffers.code_ranges, line, 0);
        count_keywords(line, &spans, &matcher, &mut file_stats, &mut Vec::new());
        assert_eq!(1, file_stats.keyword_occurences[0]);
    }

    #[test]
    fn a_keyword_counts_only_where_it_stands_as_a_word_of_its_own() {
        let line = String::from("Hello world!");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(0,0), file_stats);

        let line = String::from("class");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(1,0), file_stats);

        let line = String::from("1class");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(0,0), file_stats);

        let line = String::from("hello class word!");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(1,0), file_stats);

        let line = String::from("class class class");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(3,0), file_stats);

        let line = String::from("classclass");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(0,0), file_stats);

        let line = String::from("hello,class{word!");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(1,0), file_stats);
        
        let line = String::from("classe,");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(0,0), file_stats);
        
        let line = String::from("class interfaceclass classinterface interface");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(1,1), file_stats);
        
        let line = String::from("{class,interface}");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(1,1), file_stats);
        
        let line = String::from("{class.interface}");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(0,0), file_stats);

        // A bracket after the word belongs to the declaration, a bracket before it does not: the
        // rule and its reasons are at 'is_acceptable_before'
        let line = String::from("TFoo = class(TObject)");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(1,0), file_stats);

        let line = String::from("(class foo)");
        let mut file_stats =  FileStats::with_keywords(&[CLASS.clone(),INTERFACE.clone()]);
        keywords_of(&line, &JAVA_MATCHER, &mut file_stats);
        assert_eq!(make_file_stats(0,0), file_stats);
    }

    fn make_file_stats(class_occurances: usize, interface_occurances: usize) -> FileStats {
        fn get_keyword_map(class_occurances: usize, interface_occurances: usize) -> Vec<usize> {
            vec![class_occurances, interface_occurances]
        }

        FileStats {
            lines: 0,
            classes: LineClasses::default(),
            keyword_occurences : get_keyword_map(class_occurances, interface_occurances)
        }
    }

    #[test]
    fn a_string_delimiter_is_found_wherever_it_is_not_escaped_or_inside_another_string() {
        let single_str_opt = Some(1u8);
        let double_str_opt = Some(0u8);
        let line = String::from("Hello");
        assert_eq!(Vec::<usize>::new(),str_delimiters(&line, &PYTHON, None).0);
        let line = String::from("\"Hello\"");
        assert_eq!((vec![0,6],vec![0u8,0u8]),str_delimiters(&line, &PYTHON, None));
        let line = String::from("\"'\"Hello");
        assert_eq!((vec![0,2],vec![0u8,0u8]),str_delimiters(&line, &PYTHON, None));
        assert_eq!((vec![1,2],vec![1u8,0u8]),str_delimiters(&line, &PYTHON, single_str_opt));
        assert_eq!((vec![0,1],vec![0u8,1u8]),str_delimiters(&line, &PYTHON, double_str_opt));
        let line = String::from("''\"\"Hello");
        assert_eq!(vec![0,1,2,3],str_delimiters(&line, &PYTHON, None).0);
        assert_eq!(vec![0,1],str_delimiters(&line, &PYTHON, single_str_opt).0);
        assert_eq!(vec![2,3],str_delimiters(&line, &PYTHON, double_str_opt).0);
        let line = String::from("'\"'\"''\"He'l\"lo");
        assert_eq!(vec![0,2,3,6,9],str_delimiters(&line, &PYTHON, None).0);
        assert_eq!(vec![0,1,3,4,5,6,11],str_delimiters(&line, &PYTHON, single_str_opt).0);
        assert_eq!(vec![1,2,4,5,9,11],str_delimiters(&line, &PYTHON, double_str_opt).0);
        assert_eq!(vec![1,3,6,11],str_delimiters(&line, &JAVA, double_str_opt).0);
        let line = String::from(r#"\'\\'\\'\\\''"#);
        assert_eq!(vec![4,7,12], str_delimiters(&line, &PYTHON, None).0);
        assert_eq!(vec![4,7,12], str_delimiters(&line, &PYTHON, single_str_opt).0);
        let line = String::from(r#"["❌🔤","💭🔜","📗","📘",]"#);
        assert!(str_delimiters(&line, &PYTHON, None).0.len() == 8);
        assert!(str_delimiters(&line, &RUST, double_str_opt).0.len() == 8);
        let line = String::from(r#"[\'⣾\', '⣷', '⣯', '⣟', '⡿']"#); 
        assert!(str_delimiters(&line, &PYTHON, None).0.len() == 8);
        assert!(str_delimiters(&line, &RUST, None).0.is_empty());
        let line = String::from(r#"['⣾", '⣷", '⣯"]"#); 
        assert_eq!(vec![1u8,1u8,0u8,0u8],
                str_delimiters(&line, &PYTHON, None).1);
        let line = String::from(r#"'\'\'\''"#); 
        assert_eq!(vec![0,7], str_delimiters(&line, &PYTHON, None).0);
        let line = String::from(r#""\"\\"""#); //  """\"""
        assert_eq!(vec![0,5,6], str_delimiters(&line, &RUST, None).0);
        assert_eq!(vec![0,5,6], str_delimiters(&line, &PYTHON, None).0);
        let line = String::from(r#"\\\"\"\\""#);
        assert_eq!(vec![8], str_delimiters(&line, &RUST, None).0);
        assert_eq!(vec![8], str_delimiters(&line, &PYTHON, None).0);
    }

    // More than two string symbols, and the two rules that make them work: only the symbol that
    // opened a string closes it, and where two of them start at the same place the longer wins.
    #[test]
    fn a_language_can_declare_more_than_two_string_symbols() {
        let indices_of = |line: &str| str_delimiters(line, &PYTHON_FULL, None);

        // the third and the fourth symbol are seen at all
        assert_eq!(vec![0, 4], indices_of(r#""abc""#).0);
        assert_eq!(vec![0, 4], indices_of(r#"'abc'"#).0);

        // '"""' is one symbol and not three '"', so the docstring opens once and closes once. Its
        // number is 2, since the crossing symbols are numbered after the plain ones.
        let (indices, symbols) = indices_of(r#""""a docstring""""#);
        assert_eq!(vec![0, 14], indices);
        assert_eq!(vec![2u8, 2u8], symbols);

        // Only the symbol that opened closes: the quote of an apostrophe inside a string is text,
        // and so is a '"""' that turns up inside a plain '"'
        assert_eq!(vec![0, 10], indices_of(r#""it's fine""#).0);
        assert_eq!(vec![0, 8], indices_of(r#"'a """ b'"#).0);

        // A line that leaves one open reports its symbol, and the next line closes with that one
        let (indices, symbols) = indices_of(r#"x = """ open"#);
        assert_eq!((vec![4], vec![2u8]), (indices, symbols));
        let open = Some(2u8);
        assert_eq!(vec![5], str_delimiters("still\"\"\"", &PYTHON_FULL, open).0);
    }

    // Inside an open string the other symbol is text, and it stays text even when every occurrence
    // of the symbol that could close the string is escaped.
    #[test]
    fn the_other_symbol_stays_text_when_the_one_that_could_close_the_string_is_escaped() {
        let open_single = Some(1u8);
        let open_double = Some(0u8);

        // A '"' while a '...' string is open, and the only ''' on the line is escaped
        assert_eq!((vec![], vec![]), str_delimiters("\"\\'", &PYTHON, open_single));
        assert_eq!((vec![], vec![]), str_delimiters("'\\\"", &PYTHON, open_double));
        assert_eq!((vec![], vec![]), str_delimiters("a\"b\\'c", &PYTHON, open_single));

        // And the same line closes the string as soon as one unescaped occurrence is there
        assert_eq!(vec![3], str_delimiters("\"\\''", &PYTHON, open_single).0);
    }

    // Batch is the one shipped language that opens a comment with letters, and a language whose
    // symbols are punctuation must not start paying for the question.
    #[test]
    fn a_comment_symbol_spelled_with_letters_is_a_word_and_not_a_prefix() {
        let batch_like = Language::new("batch-like", ["bat"], StringRules::escaping_nothing(),
                ["rem", "REM", "::"], &[], []);
        let counts = |text: &str| content_counts(&parse_lines_whole(text, &batch_like));

        assert_eq!((1, 0, 1), counts("REM a comment\n"));
        assert_eq!((1, 1, 0), counts("REMOVE /Q file.txt\n"));
        assert_eq!((1, 1, 0), counts("prerem x\n"));
        // The word ends where the line does, and a symbol standing alone is still the symbol
        assert_eq!((1, 0, 1), counts("rem\n"));
        // An underscore is part of a word the way a letter is: 'REM_TEST' is a command
        assert_eq!((1, 1, 0), counts("REM_TEST /Q\n"));
        // '::' is punctuation and asks nothing, glued or not
        assert_eq!((1, 0, 1), counts("::a comment\n"));

        // The question is asked of the symbol's own ends, so nothing changes for the languages
        // whose symbols carry no letter at all
        let c_like = Language::new("c-like", ["c"], build_backslashed_quotes(), ["//"], &[("/*", "*/")], []);
        assert_eq!(vec![4], comment_delimiters("code// a comment", &c_like));
        assert_eq!(vec![0], comment_delimiters("//x", &c_like));
    }

    #[test]
    fn a_language_can_declare_more_than_two_comment_symbols() {
        let indices_of = |line: &str| comment_delimiters(line, &PYTHON_FULL);

        assert_eq!(vec![4], indices_of("code# a comment"));
        assert_eq!(vec![4], indices_of("code// a comment"));
        // the third one
        assert_eq!(vec![4], indices_of("code-- a comment"));
        // All of them on one line, in the order they are written and not in the order they are declared
        assert_eq!(vec![2, 6, 10], indices_of("a --b //c #d"));
        // and with no byte between them, where the scan has to resume past the whole of the symbol
        // it just took and not one byte into it
        assert_eq!(vec![0, 2, 3], indices_of("--#//"));
        // The same symbol twice over is the harder half of that, because the scan keeps its mark
        // per symbol: the '//' at byte 1 overlaps the one already taken and is dropped, the one at
        // byte 2 begins exactly where it ended and is kept
        assert_eq!(vec![0, 2], indices_of("////"));
    }

    // A block comment whose opening starts with the line comment symbol, which is Lua's shape
    static LUA : LazyLock<Language> = LazyLock::new(|| Language::new("lua", ["lua"],
            StringRules::escaping_with(b'\\').with_symbols(["\"", "'"]), ["--"],
            &[("--[[", "]]")], []));

    // '--[[' opens a block; it is not a '--' line comment that happens to be followed by brackets.
    // Without the longest-first rule the block never opens and its contents count as code.
    #[test]
    fn the_longer_symbol_wins_when_a_comment_and_a_block_start_together() {
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("--[[", &LUA, None, None));
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("--[[ opening", &LUA, None, None));
        // and a plain line comment still behaves like one
        assert_eq!(TextInfo::none_all(false), bounds_multi("-- just a comment", &LUA, None, None));
        assert_eq!(TextInfo::new(Some("x = 1 ".to_owned()), false, Some((0, 1)), None),
                bounds_multi("x = 1 --[[ opens here", &LUA, None, None));
        assert_eq!(TextInfo::from_slice(" y = 2"), bounds_multi("]] y = 2", &LUA, Some(0), None));
    }

    // A block comment whose opening holds the line comment symbol inside it, which is PowerShell's shape
    static POWERSHELL : LazyLock<Language> = LazyLock::new(|| Language::new("powershell", ["ps1"],
            StringRules::escaping_with(b'`').with_symbols(["\"", "'"]), ["#"], &[("<#", "#>")], []));

    // The '#' of '<#' is not a comment of its own. Reading it as one leaves the block closed for the
    // whole file, so every block comment in the language counts as code, in silence.
    #[test]
    fn a_comment_symbol_inside_the_block_opening_belongs_to_the_opening() {
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("<#", &POWERSHELL, None, None));
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("<# opening", &POWERSHELL, None, None));
        assert_eq!(TextInfo::new(Some("$x = 1 ".to_owned()), false, Some((0, 1)), None),
                bounds_multi("$x = 1 <# opens here", &POWERSHELL, None, None));
        // a plain line comment still behaves like one
        assert_eq!(TextInfo::none_all(false), bounds_multi("# just a comment", &POWERSHELL, None, None));
        // and the block closes on '#>' without its '#' reading as a comment
        assert_eq!(TextInfo::from_slice(" $y = 2"), bounds_multi("#> $y = 2", &POWERSHELL, Some(0), None));
    }

    // Two block comment pairs at once, which is Pascal's shape ('{ }' beside '(* *)') and D's
    // ('/* */' beside '/+ +/')
    static PASCAL : LazyLock<Language> = LazyLock::new(|| Language::new("pascal", ["pas"],
            StringRules::escaping_nothing().with_symbols(["'"]), ["//"],
            &[("{", "}"), ("(*", "*)")], []));

    // D's plain pair beside its nesting one, which is the shape that forces the distinction to be
    // per pair: '/* /* */' is closed in D, '/+ /+ +/' is not
    static D_LANG : LazyLock<Language> = LazyLock::new(|| Language::new("d", ["d"],
            build_backslashed_quotes(), ["//"], &[("/*", "*/")], [])
            .with_nesting_comments(&[("/+", "+/")]));

    // Lua's long bracket, one declaration covering '--[[ ]]', '--[=[ ]=]' and every level above:
    // the run of '=' is counted at the opener and only an end with the same count closes.
    static LUA_LEVELED : LazyLock<Language> = LazyLock::new(|| Language::new(
            "lua-leveled", ["lua"], StringRules::escaping_with(b'\\').with_symbols(["\"", "'"]), ["--"], &[], [])
            .with_leveled_comments(&[crate::LeveledPair::of("--[=*[", "]=*]").unwrap()]));

    #[test]
    fn a_leveled_pair_closes_only_at_an_end_carrying_the_same_count() {
        // level zero is the plain '--[[ ]]' shape
        assert_eq!(TextInfo::from_slice_w_literal("x = 1  y = "),
                bounds_multi("x = 1 --[[ note ]] y = ''", &LUA_LEVELED, None, None));
        // a ']]' inside a level-two block is text, and the block closes at ']==]'
        assert_eq!(TextInfo::none_all(false), bounds_multi("--[==[ a ]] b ]==]", &LUA_LEVELED, None, None));

        // the level crosses lines: a lower end does not close, the matching one does
        assert_eq!(TextInfo::with_open_comment_at(0, 1), bounds_multi("--[=[ open", &LUA_LEVELED, None, None));
        assert_eq!(TextInfo::with_open_comment_at(0, 1),
                bounds_multi_deep("]] not yet", &LUA_LEVELED, Some((0, 1)), None));
        assert_eq!(TextInfo::from_slice(" done"),
                bounds_multi_deep("]=] done", &LUA_LEVELED, Some((0, 1)), None));

        // '--[=' with no second bracket is no opener at all, just a line comment
        assert_eq!(TextInfo::from_slice("x = 1 "), bounds_multi("x = 1 --[= not a block", &LUA_LEVELED, None, None));

        // level zero crossing lines carries zero, which is a level and not an absence
        assert_eq!(TextInfo::with_open_comment_at(0, 0), bounds_multi("--[[ open", &LUA_LEVELED, None, None));
        assert_eq!(TextInfo::from_slice(" done"),
                bounds_multi_deep("]] done", &LUA_LEVELED, Some((0, 0)), None));
        // and a language whose only pair is the leveled one still takes the multiline path
        assert!(LUA_LEVELED.supports_multiline_comments());
    }

    static OCAML : LazyLock<Language> = LazyLock::new(|| Language::new(
            "ocaml-like", ["ml"], StringRules::escaping_with(b'\\').with_multiline_strings(["\""]),
            [""; 0], &[], [])
            .with_nesting_comments(&[("(*", "*)")]));

    #[test]
    fn a_nesting_pair_closes_only_when_as_many_ends_as_starts_have_passed() {
        assert_eq!(TextInfo::from_slice(" d"),
                bounds_multi("(* a (* b *) c *) d", &OCAML, None, None));
        assert_eq!(TextInfo::with_open_comment_at(0, 2), bounds_multi("(* one (* two", &OCAML, None, None));
        // an end on a later line closes one level and the block stays open
        assert_eq!(TextInfo::with_open_comment_at(0, 1),
                bounds_multi_deep("still *) inside", &OCAML, Some((0, 2)), None));
        assert_eq!(TextInfo::from_slice(" x"), bounds_multi_deep("done *) x", &OCAML, Some((0, 1)), None));
        // a deeper start on a passing line deepens the carried state
        assert_eq!(TextInfo::with_open_comment_at(0, 3),
                bounds_multi_deep("more (* here", &OCAML, Some((0, 2)), None));
    }

    #[test]
    fn the_plain_pair_of_a_language_does_not_nest_while_its_nesting_pair_does() {
        // D's '/*' still closes at the first '*/', nested-looking or not
        assert_eq!(TextInfo::from_slice(" tail */"),
                bounds_multi("/* a /* b */ tail */", &D_LANG, None, None));
        // its '/+' counts depth
        assert_eq!(TextInfo::from_slice(" d"), bounds_multi("/+ a /+ b +/ c +/ d", &D_LANG, None, None));
        assert_eq!(TextInfo::with_open_comment_at(1, 2), bounds_multi("/+ one /+ two", &D_LANG, None, None));
    }

    #[test]
    fn a_second_comment_pair_opens_and_only_its_own_end_closes_it() {
        assert_eq!(TextInfo::none_all(false), bounds_multi("{ comment }", &PASCAL, None, None));
        assert_eq!(TextInfo::none_all(false), bounds_multi("(* comment *)", &PASCAL, None, None));
        assert_eq!(TextInfo::from_slice_w_literal("x := 1; "),
                bounds_multi("x := 1; { note } '4'", &PASCAL, None, None));

        // the other pair's end inside a block is text, and the block still closes with its own
        assert_eq!(TextInfo::none_all(false), bounds_multi("{ close with *) no, with }", &PASCAL, None, None));
        assert_eq!(TextInfo::none_all(false), bounds_multi("(* a } inside *)", &PASCAL, None, None));

        // a block left open remembers which pair opened it, across lines as well
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("{ open", &PASCAL, None, None));
        assert_eq!(TextInfo::with_open_comment(1), bounds_multi("(* open", &PASCAL, None, None));
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("still *) going", &PASCAL, Some(0), None));
        assert_eq!(TextInfo::with_open_comment(1), bounds_multi("still } going", &PASCAL, Some(1), None));
        assert_eq!(TextInfo::from_slice(" x := 2;"), bounds_multi("} x := 2;", &PASCAL, Some(0), None));
        assert_eq!(TextInfo::from_slice(" x := 2;"), bounds_multi("*) x := 2;", &PASCAL, Some(1), None));

        // one line with both pairs in turn, and the code between them kept. Whitespace alone is
        // not code, so the second reads as a line holding a literal and nothing else.
        assert_eq!(TextInfo::from_slice_w_literal(" x "),
                bounds_multi("{ a } x (* b *) ''", &PASCAL, None, None));
        assert_eq!(TextInfo::none_all(true), bounds_multi("{ a } (* b *) ''", &PASCAL, None, None));
        // the '{' block swallows a '(*' opener sitting inside it
        assert_eq!(TextInfo::from_slice(" c"), bounds_multi("{ a (* b } c", &PASCAL, None, None));
    }

    #[test]
    fn the_d_shape_where_both_pairs_share_a_first_byte_still_matches_by_pair() {
        assert_eq!(TextInfo::none_all(false), bounds_multi("/* comment */", &D_LANG, None, None));
        assert_eq!(TextInfo::none_all(false), bounds_multi("/+ comment +/", &D_LANG, None, None));
        // '*/' does not close a '/+' block, '+/' does not close a '/*' block. Either way the open
        // pair survives the line, and either kind of pair leaves a comment line behind it.
        assert_eq!(TextInfo::with_open_comment(1), bounds_multi("/+ a */ still open", &D_LANG, None, None));
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("/* a +/ still open", &D_LANG, None, None));
        // and across lines
        assert_eq!(TextInfo::with_open_comment(1), bounds_multi("text */ text", &D_LANG, Some(1), None));
        assert_eq!(TextInfo::from_slice(" code"), bounds_multi("+/ code", &D_LANG, Some(1), None));
        assert_eq!(TextInfo::from_slice(" a  b"), bounds_multi("/* x */ a /+ y +/ b", &D_LANG, None, None));
    }

    // Strings that open with one symbol and close with another: Rust's raw form and C#'s verbatim
    // form. Their pairs ride beside the ordinary quotes, and inside them nothing escapes.
    static RUST_RAW : LazyLock<Language> = LazyLock::new(|| Language::new(
            "rust-raw", ["rs"], StringRules::escaping_with(b'\\').with_multiline_strings(["\""])
                    .with_string_pairs(&[("r#\"", "\"#")]),
            ["//"], &[("/*", "*/")], []));

    static CSHARP_VERBATIM : LazyLock<Language> = LazyLock::new(|| Language::new(
            "csharp-verbatim", ["cs"], build_backslashed_quotes().with_multiline_strings(["\"\"\""])
                    .with_string_pairs(&[("@\"", "\"")]),
            ["//"], &[("/*", "*/")], []));

    // The shape of the shipped Rust file: a crossing quote, and the character literal beside it
    static RUST_CHARS : LazyLock<Language> = LazyLock::new(|| Language::new(
            "rust-chars", ["rs"], StringRules::escaping_with(b'\\').with_char_literals(["'"])
                    .with_multiline_strings(["\""]),
            ["//"], &[("/*", "*/")], []));

    // A character literal that does not close on its own line is not a literal at all, which keeps
    // a lifetime's lone ' from swallowing the rest of its line. One that does close shields what it
    // holds, which keeps the quote of '"' from opening a string that never closes.
    #[test]
    fn a_character_literal_pairs_on_its_own_line_or_is_not_a_literal_at_all() {
        // the quote inside the literal opens nothing, so nothing is carried to the next line
        assert_eq!(TextInfo::from_slice_w_literal("let c = ;"),
                bounds_multi("let c = '\"';", &RUST_CHARS, None, None));
        // a lone ' is a lifetime, not an open literal: the whole line is plain code
        assert_eq!(TextInfo::from_slice("let x: &'a str = y;"),
                bounds_multi("let x: &'a str = y;", &RUST_CHARS, None, None));
        // two lone ticks on one line do not pair either, because what sits between them is not one
        // character
        assert_eq!(TextInfo::from_slice("fn get<'a>(x: &'a str) -> &'a str {"),
                bounds_multi("fn get<'a>(x: &'a str) -> &'a str {", &RUST_CHARS, None, None));
        assert_eq!(TextInfo::from_slice_w_literal("let msg: &'static str = ;"),
                bounds_multi("let msg: &'static str = \"don't panic\";", &RUST_CHARS, None, None));
        // and an escape sequence of any length is still one character
        assert_eq!(TextInfo::from_slice_w_literal("let u = ;"),
                bounds_multi("let u = '\\u{1F600}';", &RUST_CHARS, None, None));
        // escapes inside the literal behave as in any string: '\'' and '\\' close where Rust says
        assert_eq!(TextInfo::from_slice_w_literal("let q = ;"),
                bounds_multi("let q = '\\'';", &RUST_CHARS, None, None));
        assert_eq!(TextInfo::from_slice_w_literal("let b = ;"),
                bounds_multi("let b = '\\\\';", &RUST_CHARS, None, None));
        // inside a comment or a string the symbol is not a literal
        assert_eq!(TextInfo::none_all(false), bounds_multi("// don't", &RUST_CHARS, None, None));
        assert_eq!(TextInfo::from_slice_w_literal("let s = ;"),
                bounds_multi("let s = \"don't\";", &RUST_CHARS, None, None));
        // and a crossing quote left open from an earlier line is closed by its own symbol, with
        // the literal's halves inside it read as text
        assert_eq!(TextInfo::new(Some(" after".to_owned()), true, None, None),
                bounds_multi("tick ' text\" after", &RUST_CHARS, None, Some(1)));
    }

    // The symbol is found by the quote the language already declares and checked backwards, so
    // declaring the pair costs nothing per line: measured same-binary with the pair declared and
    // not, 1.01 ± 0.08 over a Rust-heavy tree.
    #[test]
    fn a_symbol_led_by_a_letter_is_searched_by_a_byte_the_scan_wanted_anyway() {
        let plan = ScanPlan::build(&RUST_RAW);
        assert!(plan.chunks.iter().all(|c| !c.bytes[..c.len as usize].contains(&b'r')),
                "the scan searches for 'r', which floods on ordinary code");
        // '"', '/' and '*' cover everything the language declares, in one pass
        assert_eq!(1, plan.chunks.len());

        // The byte is chosen from the ones the language already looks for and not from the end of
        // the symbol: on its last punctuation, C++'s 'R"(' would be found by '('
        let cpp_raw = Language::new("cpp-like", ["cpp"],
                build_backslashed_quotes().with_string_pairs(&[("R\"(", ")\"")]),
                ["//"], &[("/*", "*/")], []);
        let plan = ScanPlan::build(&cpp_raw);
        assert!(plan.chunks.iter().all(|c| !c.bytes[..c.len as usize].contains(&b'(')),
                "the opener is searched by '(' in a language made of brackets");
    }

    // C++'s ')"' would put the bracket that stands in front of every call into the scan, while the
    // quote it also holds is searched anyway: declaring the raw pair adds no byte and no pass.
    #[test]
    fn a_symbol_is_searched_by_a_byte_another_symbol_needs_before_one_of_its_own() {
        let plain = Language::new("cpp-like", ["cpp"], build_backslashed_quotes(), ["//"], &[("/*", "*/")], []);
        let with_raw = Language::new("cpp-like", ["cpp"],
                build_backslashed_quotes().with_string_pairs(&[("R\"(", ")\"")]),
                ["//"], &[("/*", "*/")], []);

        let bytes_of = |language: &Language| {
            let plan = ScanPlan::build(language);
            let mut bytes = plan.chunks.iter()
                    .flat_map(|c| c.bytes[..c.len as usize].to_vec()).collect::<Vec<u8>>();
            bytes.sort_unstable();
            (bytes, plan.chunks.len())
        };
        assert_eq!(bytes_of(&plain), bytes_of(&with_raw));
        // '*' is dropped rather than kept: '*/' holds the '/' that '//' forces into the scan, and
        // every pointer, product and comment continuation line stops being a candidate position
        assert_eq!((vec![b'"', b'/'], 1), bytes_of(&with_raw));
    }

    // Whether a backslash cancels the symbol after it is a fact about the language and not about
    // the symbol, which is why the same declaration cannot serve both shells and PowerShell: both
    // write a quote that escapes nothing inside itself, and only one of them escapes outside one.
    #[test]
    fn what_cancels_a_symbol_is_read_from_the_language_and_not_assumed() {
        let shell = Language::new("shell-like", ["sh"],
                StringRules::escaping_with(b'\\').with_raw_multiline_strings(["'"]), ["#"], &[], []);
        let powershell = Language::new("ps-like", ["ps1"],
                StringRules::escaping_with(b'`').with_raw_multiline_strings(["'"]), ["#"], &[], []);
        let pascal = Language::new("pascal-like", ["pas"],
                StringRules::escaping_nothing().with_symbols(["'"]), ["//"], &[], []);

        // the shell escapes with the backslash, so the apostrophe opens nothing
        assert_eq!(TextInfo::from_slice(r"echo I\'m done"),
                bounds_multi(r"echo I\'m done", &shell, None, None));
        // inside the string it is a byte again, so the same shape closes one
        assert_eq!(TextInfo::from_slice_w_literal("echo  done"),
                bounds_multi(r"echo 'a\' done", &shell, None, None));

        // PowerShell escapes with the backtick, so a path ending in a backslash opens a string
        // where the shell would not
        assert_eq!(TextInfo::from_slice_w_literal(r"cd C:\"),
                bounds_multi(r"cd C:\'Program Files'", &powershell, None, None));

        // and a language that escapes by doubling its quote cancels nothing: the string closes at
        // its own quote, which is what leaves the comment after it a comment and not string content
        assert_eq!(TextInfo::from_slice_w_literal("s := ; "),
                bounds_multi(r"s := 'C:\'; // a comment", &pascal, None, None));
    }

    // C++'s raw string crosses lines by itself and keeps everything inside it: without the pair
    // declared, a file that opens one and writes a quote, a comment opener or a bracket inside it
    // counts the rest of the line as whatever those symbols say.
    #[test]
    fn a_cpp_raw_string_keeps_the_quotes_and_brackets_inside_it() {
        let cpp = Language::new("cpp-like", ["cpp"],
                build_backslashed_quotes().with_string_pairs(&[("R\"(", ")\"")]),
                ["//"], &[("/*", "*/")], []);

        assert_eq!(TextInfo::from_slice_w_literal("auto s = ;"),
                bounds_multi(r#"auto s = R"(say "hi" // and (stay) code)";"#, &cpp, None, None));
        // a prefixed form is the same three bytes behind a letter or two, so one declaration
        // answers for 'LR"(', 'uR"(' and 'u8R"(' as well
        assert_eq!(TextInfo::from_slice_w_literal("auto s = u8;"),
                bounds_multi(r#"auto s = u8R"(text)";"#, &cpp, None, None));

        // the closer is the bracket every call in the language ends with, and standing in code with
        // nothing open it is text: this is one ordinary string and not the end of anything
        assert_eq!(TextInfo::from_slice_w_literal("printf();"),
                bounds_multi(r#"printf(")");"#, &cpp, None, None));

        // left open it reports its own symbol, an ordinary quote cannot close it on the next line,
        // and its own closer can
        assert_eq!(TextInfo::new(Some("auto s = ".to_owned()), true, None, Some(1)),
                bounds_multi(r#"auto s = R"(open"#, &cpp, None, None));
        assert_eq!(TextInfo::new(None, true, None, Some(1)),
                bounds_multi(r#"still text "quoted" // not a comment"#, &cpp, None, Some(1)));
        assert_eq!(TextInfo::new(Some(";".to_owned()), true, None, None),
                bounds_multi(r#"done)";"#, &cpp, None, Some(1)));
    }

    // Rust's shortest raw form is one letter and a quote, so every string ending in 'r' carries
    // what looks like an opener: '"abcr"' must still be one plain string and not an opener inside
    // one. What saves it is that the pair cannot open while another string is open.
    #[test]
    fn a_raw_opener_that_appears_inside_an_ordinary_string_is_text() {
        let rust = Language::new("rust-like", ["rs"],
                StringRules::escaping_with(b'\\').with_multiline_strings(["\""])
                        .with_string_pairs(&[("r\"", "\""), ("r#\"", "\"#")]),
                ["//"], &[("/*", "*/")], []);

        assert_eq!(TextInfo::from_slice_w_literal("let s = ;"),
                bounds_multi(r#"let s = "abcr";"#, &rust, None, None));
        assert_eq!(TextInfo::from_slice_w_literal("let p = ;"),
                bounds_multi(r#"let p = r"C:\temp\";"#, &rust, None, None));
        // a quote inside the one-letter form does end it, which is what that form means in Rust
        assert_eq!(TextInfo::from_slice_w_literal("let q = ab;"),
                bounds_multi(r#"let q = r"a"ab"b";"#, &rust, None, None));
    }

    #[test]
    fn a_string_that_opens_with_one_symbol_closes_only_with_its_other_half() {
        // the quotes inside the raw string are text; read as ordinary quotes they leave one open
        // and everything after this line counts as string content
        assert_eq!(TextInfo::from_slice_w_literal("let a = ; done"),
                bounds_multi(r##"let a = r#"say "hi"#; done"##, &RUST_RAW, None, None));

        // a raw string left open reports its own symbol, an ordinary quote cannot close it on the
        // next line, and its own closer can
        assert_eq!(TextInfo::new(Some("x = ".to_owned()), true, None, Some(1)),
                bounds_multi(r##"x = r#"open"##, &RUST_RAW, None, None));
        assert_eq!(TextInfo::new(None, true, None, Some(1)),
                bounds_multi(r#"say "quoted" more"#, &RUST_RAW, None, Some(1)));
        assert_eq!(TextInfo::none_all(true), bounds_multi(r##"done"#"##, &RUST_RAW, None, Some(1)));

        // a closer with nothing open is not a delimiter: the quote of '"#"' opens an ordinary
        // string holding a '#', which is what that line means in Rust
        assert_eq!(TextInfo::from_slice_w_literal("let s = ;"),
                bounds_multi(r##"let s = "#";"##, &RUST_RAW, None, None));
    }

    #[test]
    fn inside_a_two_sided_pair_the_backslash_does_not_escape() {
        // a raw string body ending in a backslash still closes: nothing escapes inside the pair
        assert_eq!(TextInfo::from_slice_w_literal("let p = ;"),
                bounds_multi(r##"let p = r#"C:\path\"#;"##, &RUST_RAW, None, None));
        // while the ordinary quote keeps the escape rule, so an escaped quote leaves it open
        assert_eq!(TextInfo::new(Some("let q = ".to_owned()), true, None, Some(0)),
                bounds_multi(r#"let q = "C:\path\";"#, &RUST_RAW, None, None));

        // C#'s verbatim string closes at the plain quote its pair declares, backslash and all
        assert_eq!(TextInfo::from_slice_w_literal("var s =  + x;"),
                bounds_multi(r#"var s = @"C:\temp\" + x;"#, &CSHARP_VERBATIM, None, None));
    }

    // One symbol at both ends says nothing about whether a backslash cancels it, and reading it off
    // that shape leaves every 'C:\' open to the end of the file in Go, Odin and D. The backtick
    // escapes nothing in those three and does escape in a JavaScript template literal, so the two
    // halves of this test are the same line in two languages with opposite right answers.
    #[test]
    fn a_one_sided_form_escapes_or_not_as_the_language_declares_and_not_as_its_shape_suggests() {
        let go = Language::new("go-like", ["go"],
                build_backslashed_quotes().with_raw_multiline_strings(["`"]), ["//"], &[("/*", "*/")], []);
        let js = Language::new("js-like", ["js"],
                build_backslashed_quotes().with_multiline_strings(["`"]), ["//"], &[("/*", "*/")], []);

        assert_eq!(TextInfo::from_slice_w_literal("var sep = ;"),
                bounds_multi(r"var sep = `C:\`;", &go, None, None));
        assert_eq!(TextInfo::new(Some("var sep = ".to_owned()), true, None, Some(1)),
                bounds_multi(r"var sep = `C:\`;", &js, None, None));

        // and the raw form is a string in every other way: its own symbol closes it, an ordinary
        // quote inside it is text, and a comment opener inside it opens nothing
        assert_eq!(TextInfo::new(Some("var s = ".to_owned()), true, None, Some(1)),
                bounds_multi("var s = `open", &go, None, None));
        assert_eq!(TextInfo::none_all(true), bounds_multi("still \" /* text `", &go, None, Some(1)));
    }

    // The languages a section can resolve to, keyed the way the real run keys them: definitions by
    // name, and the attribute values by extension.
    fn section_fixture() -> (HashMap<String, Language>, HashMap<String, Arc<str>>) {
        let js = Language::new("JS", ["js"], build_backslashed_quotes(), ["//"], &[("/*", "*/")],
                [Keyword { descriptive_name: "functions".to_owned(), aliases: vec!["function".to_owned()] }]);
        let css = Language::new("CSS", ["css"], StringRules::escaping_nothing(), [""; 0], &[("/*", "*/")], []);
        let languages = crate::languages::keyed_by_name(vec![js, css]);
        let extensions = HashMap::from([("js".to_owned(), Arc::from("JS")), ("css".to_owned(), Arc::from("CSS"))]);
        (languages, extensions)
    }

    fn web_shell() -> Language {
        Language::new("web", ["wbl"], StringRules::escaping_nothing(), [""; 0], &[("<!--", "-->")], [])
                .with_nested_languages(&[NestedLanguage::of("<script", "</script>", "js"),
                        NestedLanguage::of("<style", "</style>", "css")])
    }

    fn parse_with_sections(contents: &str, shell: &Language,
        languages: &HashMap<String, Language>, extensions: &HashMap<String, Arc<str>>) -> FileReport
    {
        let lookup = NestedLanguageLookup { languages, extension_to_name: extensions, set_aside: &NO_SET_ASIDE };
        parse_lines::<false>(contents, shell, &lookup, &mut KeywordMatchers::default(),
                &EngineConfig::default(), &mut ParseBuffers::default(), &mut ExplainLog::default())
    }

    #[test]
    fn a_section_is_counted_with_its_own_language_and_the_tag_lines_stay_with_the_shell() {
        let (languages, extensions) = section_fixture();
        let contents = "<p>hello</p>\n<script>\n// a js comment\nvar s = \"x\"; function f() {}\n</script>\n\
                <style>\n/* css comment */\n</style>\n<p>bye</p>\n";

        let report = parse_with_sections(contents, &web_shell(), &languages, &extensions);
        assert_eq!((6, 6, 0), content_counts(&report.shell),
                "the tag lines and the html around them belong to the shell");

        let js = &report.sections[0];
        assert_eq!("JS", js.language.as_str());
        assert_eq!((2, 1, 1), content_counts(&js.stats));
        assert_eq!(vec![1], js.stats.keyword_occurences, "the js keywords count inside the js section");
        let css = &report.sections[1];
        assert_eq!("CSS", css.language.as_str());
        assert_eq!((1, 0, 1), content_counts(&css.stats));

        // the bytes of a section are exactly the bytes between its tag lines
        let js_bytes = contents.find("</script>").unwrap() - (contents.find("<script>").unwrap() + "<script>\n".len());
        assert_eq!(js_bytes, js.bytes);
        assert_eq!(contents.lines().count(), report.total_lines(), "a line of the file is counted exactly once");
    }

    // The opener only counts where the shell read it as code: inside a comment or a string of the
    // shell it is text, which is what tokei gets only half right
    #[test]
    fn an_opener_inside_a_comment_or_a_string_of_the_shell_opens_nothing() {
        let (languages, extensions) = section_fixture();
        let report = parse_with_sections("<!-- <script> -->\n<p>x</p>\n", &web_shell(), &languages, &extensions);
        assert!(report.sections.is_empty(), "a tag inside a comment opened a section");
        assert_eq!((2, 1, 1), content_counts(&report.shell));

        let stringy = Language::new("webstr", ["wbs"], build_backslashed_quotes(), [""; 0], &[], [])
                .with_nested_languages(&[NestedLanguage::of("<script", "</script>", "js")]);
        let report = parse_with_sections("x = \"<script>\"\n", &stringy, &languages, &extensions);
        assert!(report.sections.is_empty(), "a tag inside a string opened a section");
    }

    #[test]
    fn the_tag_names_its_language_and_falls_to_the_declared_default_when_it_does_not() {
        let (languages, extensions) = section_fixture();
        let shell = web_shell();

        // 'lang' wins over the region's default, however the value is quoted
        for tag in ["<script lang=\"css\">", "<script lang='css'>", "<script lang=css>"] {
            let contents = format!("{tag}\n/* x */\n</script>\n");
            let report = parse_with_sections(&contents, &shell, &languages, &extensions);
            assert_eq!("CSS", report.sections[0].language, "{tag} did not resolve its language");
        }
        // a mime 'type' names its language after the slash, by extension or by the language's own
        // name, since people write both and only one of the two is an extension
        let report = parse_with_sections("<script type=\"text/js\">\nvar x = 1;\n</script>\n", &shell, &languages, &extensions);
        assert_eq!("JS", report.sections[0].language);
        let report = parse_with_sections("<style lang=\"CSS\">\n.a { color: red; }\n</style>\n", &shell, &languages, &extensions);
        assert_eq!("CSS", report.sections[0].language, "a language's own name was not recognised");
        // a value nobody recognises falls to the default rather than losing the section
        let report = parse_with_sections("<script lang=\"zz\">\nvar x = 1;\n</script>\n", &shell, &languages, &extensions);
        assert_eq!("JS", report.sections[0].language);
        // and 'slang=' is not 'lang='
        let report = parse_with_sections("<script slang=\"css\">\nvar x = 1;\n</script>\n", &shell, &languages, &extensions);
        assert_eq!("JS", report.sections[0].language);
    }

    #[test]
    fn tags_match_in_any_case() {
        let (languages, extensions) = section_fixture();
        let report = parse_with_sections("<SCRIPT>\n// x\n</SCRIPT>\n<p>y</p>\n", &web_shell(), &languages, &extensions);
        assert_eq!(1, report.sections.len(), "an upper case tag was not read as a tag");
        assert_eq!((1, 0, 1), content_counts(&report.sections[0].stats));

        let contents = "<script>\n// x\n</SCRIPT>\n";
        let report = parse_with_sections(contents, &web_shell(), &languages, &extensions);
        assert_eq!(1, report.sections.len(), "a closer in another case did not end the section");
        let section_from = contents.find("// x").unwrap();
        assert_eq!(contents.find("</SCRIPT>").unwrap() - section_from, report.sections[0].bytes);
    }

    // Nothing forces an opener to be a tag rather than the same word written as text, so a section
    // that never closes is text: without this, one '<script' in a paragraph hands every line under
    // it to another language, and the file it costs is the whole of it
    #[test]
    fn a_section_that_never_closes_stays_with_the_shell() {
        let (languages, extensions) = section_fixture();
        let report = parse_with_sections("<p>x</p>\n<script>\n// one\n// two\n", &web_shell(), &languages, &extensions);
        assert!(report.sections.is_empty(), "an unclosed opener took the rest of the file");
        assert_eq!(4, report.shell.lines);

        // The word has to end where a tag name ends, so a longer word beginning with it is text
        let report = parse_with_sections("<scriptures>\n// one\n</scriptures>\n", &web_shell(), &languages, &extensions);
        assert!(report.sections.is_empty(), "a longer word beginning with the tag opened a section");

        // And the shell keeps reading the lines it kept, with its own symbols
        let report = parse_with_sections("<p>x</p>\n<script>\n<!-- a note -->\n", &web_shell(), &languages, &extensions);
        assert_eq!((3, 2, 1), content_counts(&report.shell));
    }

    // The two halves of one pair are read by one rule: the opener already ends where a tag name
    // ends, and so does the closer. A browser was asked the same three questions and gave these
    // three answers.
    #[test]
    fn a_closing_tag_ends_a_section_wherever_html_says_it_does() {
        let (languages, extensions) = section_fixture();
        let closed_by = |closer: &str| parse_with_sections(
                &format!("<script>\nvar x = 1;\n{closer}\n<p>y</p>\n"), &web_shell(), &languages, &extensions);

        for closer in ["</script>", "</script >", "</script   >", "</SCRIPT >", "</script foo>"] {
            let report = closed_by(closer);
            assert_eq!(1, report.sections.len(), "'{closer}' closed no section");
            assert_eq!((1, 1, 0), content_counts(&report.sections[0].stats), "'{closer}'");
            assert_eq!(3, report.shell.lines, "'{closer}' left the wrong lines to the shell");
        }

        // A longer name is another tag and closes nothing, so the section never closes and the
        // file stays what it was
        assert!(closed_by("</scriptfoo>").sections.is_empty());

        // Neither does a name that never reaches its '>' on the line it began
        assert!(closed_by("</script").sections.is_empty());
    }

    #[test]
    fn what_cannot_be_a_section_counts_as_the_shell_it_always_was() {
        let (languages, extensions) = section_fixture();
        let report = parse_with_sections("<script\nlang=\"js\">\nvar x = 1;\n</script>\n", &web_shell(), &languages, &extensions);
        assert!(report.sections.is_empty(), "a tag split over two lines opened a section");

        let report = parse_with_sections("<script>var x = 1;</script>\n<p>y</p>\n", &web_shell(), &languages, &extensions);
        assert!(report.sections.is_empty(), "a one line section left the line");
        assert_eq!((2, 2, 0), content_counts(&report.shell));

        // A default nothing can answer for, by extension or by name, leaves the section as shell
        // rather than counting it under a language that does not exist
        let unknown = Language::new("web", ["wbl"], StringRules::escaping_nothing(), [""; 0], &[("<!--", "-->")], [])
                .with_nested_languages(&[NestedLanguage::of("<script", "</script>", "nosuchthing")]);
        let report = parse_with_sections("<script>\nvar x = 1;\n</script>\n", &unknown, &languages, &extensions);
        assert!(report.sections.is_empty(), "a section resolved to a language nothing declares");
        assert_eq!((3, 3, 0), content_counts(&report.shell));
    }

    #[test]
    fn two_sections_of_the_same_language_are_one_entry_of_the_report() {
        let (languages, extensions) = section_fixture();
        let contents = "<script>\n// one\n</script>\n<script>\n// two\nvar x = 1;\n</script>\n";
        let report = parse_with_sections(contents, &web_shell(), &languages, &extensions);
        assert_eq!(1, report.sections.len());
        assert_eq!((3, 1, 2), content_counts(&report.sections[0].stats));
    }

    // A string ends with its line unless its symbol was declared to cross lines.
    #[test]
    fn an_unbalanced_quote_costs_its_line_and_not_the_rest_of_the_file() {
        let plain = Language::new("py-like", ["py"], StringRules::escaping_with(b'\\')
                .with_symbols(["\"", "'"]).with_multiline_strings(["\"\"\""]), ["#"], &[], []);
        let crossing = Language::new("py-like", ["py"], StringRules::escaping_with(b'\\')
                .with_multiline_strings(["\"\"\"", "\"", "'"]), ["#"], &[], []);
        let contents = "a = \"unbalanced\nb = 1\nc = 2\n# comment\n";

        let stats = parse_lines_whole(contents, &plain);
        assert_eq!((4, 3, 1), content_counts(&stats));
        // declared crossing, everything after the quote is string content and code to the end
        let stats = parse_lines_whole(contents, &crossing);
        assert_eq!((4, 4, 0), content_counts(&stats));

        // and the docstring symbol, which is declared crossing in both, still spans lines
        let doc = "d = \"\"\"docstring\n# still string\n\"\"\"\ne = 1\n# comment\n";
        let stats = parse_lines_whole(doc, &plain);
        assert_eq!((5, 4, 1), content_counts(&stats));
    }

    // A closer of more than one byte has to be stepped over whole, or the tail of a '"""' lands in
    // the code text of the line.
    #[test]
    fn closing_a_string_advances_past_the_whole_closing_symbol() {
        assert_eq!(TextInfo::from_slice_w_literal("var d =  y"),
                bounds_multi(r#"var d = """doc""" y"#, &CSHARP_VERBATIM, None, None));
        assert_eq!(TextInfo::from_slice_w_literal("x =  y"),
                bounds_multi(r#"x = """doc""" y"#, &PYTHON_FULL, None, None));
    }

    static DEFN : LazyLock<Keyword> = LazyLock::new(|| Keyword::new("functions", ["(defn", "defn"]));

    static CLOJURE : LazyLock<Language> = LazyLock::new(|| Language::new("clojure", ["clj"],
            build_backslashed_quotes(), [";"], &[], [DEFN.clone()]));

    // '(' is not an accepted boundary, so the Lisp family's '(defn' counts zero through a bare
    // alias and the bracket has to belong to the alias. The two forms cannot double count: wherever
    // '(defn' matches, the bare 'defn' sits one byte later with the bracket before it, and is
    // rejected for that.
    #[test]
    fn a_bracketed_alias_counts_once_and_never_twice() {
        let matcher = KeywordMatcher::build(&CLOJURE).unwrap();
        let count_of = |line: &str| {
            let mut file_stats = FileStats::with_keywords(std::slice::from_ref(&DEFN));
            keywords_of(line, &matcher, &mut file_stats);
            file_stats.keyword_occurences[0]
        };

        assert_eq!(1, count_of("(defn foo [x] x)"));
        assert_eq!(1, count_of("  (defn foo [x] x)"));
        assert_eq!(1, count_of("(do (defn foo))"));
        // the bare form still counts where nothing precedes it
        assert_eq!(1, count_of("defn"));
        assert_eq!(1, count_of("defn foo"));
        // and neither form fires on a longer word
        assert_eq!(0, count_of("(defnx foo)"));
        assert_eq!(0, count_of("(mydefn foo)"));
    }

    // Every shipped alias ends in a letter or a dot, which is not accepted in front of a keyword,
    // so a run of them already answered zero and no shipped count can move. The synthetic alias
    // here is the one shape whose own bytes are accepted on both sides, and it is the whole of
    // what the rule decides.
    #[test]
    fn a_run_of_touching_aliases_counts_as_nothing() {
        let clojure = KeywordMatcher::build(&CLOJURE).unwrap();
        let defns = |line: &str| {
            let mut file_stats = FileStats::with_keywords(std::slice::from_ref(&DEFN));
            keywords_of(line, &clojure, &mut file_stats);
            file_stats.keyword_occurences[0]
        };

        assert_eq!(0, defns("(defn(defn"));
        assert_eq!(0, defns("(defn(defn(defn"));
        assert_eq!(0, defns("(defn(defn(defn(defn"));
        assert_eq!(0, defns("(defn(defn(defn(defn(defn"));
        assert_eq!(2, defns("(defn (defn"));
        assert_eq!(1, defns("(defn(defn (defn"));

        let braced = Keyword::new("braced", ["{x{"]);
        let language = Language::new("braced", ["bx"], build_backslashed_quotes(), [";"], &[],
                [braced.clone()]);
        let matcher = KeywordMatcher::build(&language).unwrap();
        let braces = |line: &str| {
            let mut file_stats = FileStats::with_keywords(std::slice::from_ref(&braced));
            keywords_of(line, &matcher, &mut file_stats);
            file_stats.keyword_occurences[0]
        };

        assert_eq!(1, braces("{x{"));
        assert_eq!(2, braces("{x{ {x{"));
        let chains = (2..=7).map(|n| braces(&"{x{".repeat(n))).collect::<Vec<usize>>();
        assert_eq!(vec![0, 0, 0, 0, 0, 0], chains);
        assert_eq!(1, braces("{x{{x{ {x{"));
    }

    // Every symbol of a kind has to be searched in the same pass, otherwise its positions would not
    // come out in the order they appear on the line and the merge below would read them out of turn.
    #[test]
    fn every_kind_is_searched_whole_in_a_single_pass() {
        for language in [&*JAVA, &*RUST, &*PHP, &*PYTHON, &*PYTHON_FULL, &*PASCAL, &*D_LANG, &*LUA, &*POWERSHELL] {
            let plan = ScanPlan::build(language);
            let searched = |byte: u8| plan.chunks.iter().filter(|c| c.bytes[..c.len as usize].contains(&byte)).count();
            // The byte a symbol is found by, which is its first only until it is anchored on another
            let bytes_of = |kind: u8| plan.slots.iter().enumerate().filter(|(_, slot)| slot.kind == kind)
                    .map(|(at, slot)| plan.symbols[at][slot.anchor as usize]).collect::<Vec<u8>>();

            for kind in [STRINGS, COMMENTS, COM_STARTS, COM_ENDS] {
                let bytes = bytes_of(kind);
                if bytes.is_empty() { continue; }
                let holding = plan.chunks.iter()
                        .filter(|c| bytes.iter().any(|b| c.bytes[..c.len as usize].contains(b)))
                        .count();
                assert_eq!(1, holding, "{} splits a kind across passes", language.name);
                // and no byte is looked for twice, which would report the same symbol from two passes
                for byte in bytes {
                    assert_eq!(1, searched(byte), "{} searches a byte twice", language.name);
                }
            }
        }
    }

    #[test]
    fn a_language_is_scanned_in_as_few_passes_as_its_first_bytes_allow() {
        // '"', '/' and '*' cover the string, the comment and both multiline symbols
        assert_eq!(1, ScanPlan::build(&JAVA).chunks.len());
        // '"', '\'' and '#' cover everything python declares
        assert_eq!(1, ScanPlan::build(&PYTHON).chunks.len());
        // php needs '"', '\'', '#', '/' and '*', which is two passes and not five
        assert_eq!(2, ScanPlan::build(&PHP).chunks.len());
    }

    // A symbol may not overlap itself: the candidate positions of '/' in "///" are 0, 1 and 2, but
    // only 0 begins a comment.
    #[test]
    fn a_symbol_does_not_overlap_itself() {
        assert_eq!(vec![0], comment_delimiters("///", &JAVA));
        assert_eq!(vec![0], comment_delimiters("//", &JAVA));
        assert_eq!(vec![0, 2], comment_delimiters("////", &JAVA));
        assert_eq!(vec![1], comment_delimiters("a///", &JAVA));

        // the same for a string symbol longer than one byte: six quotes are two '"""' and not four,
        // and five are one '"""' that never closes
        assert_eq!(vec![0, 3], str_delimiters(&"\"".repeat(6), &PYTHON_FULL, None).0);
        assert_eq!(vec![0], str_delimiters(&"\"".repeat(5), &PYTHON_FULL, None).0);
    }

    // Nothing about the lines may differ from 'str::lines', so the standard library is the oracle
    // here: every shape that behaves differently at the end of a file.
    #[test]
    fn the_line_iterator_agrees_with_the_standard_library() {
        let cases = ["", "\n", "\n\n", "a", "a\n", "a\nb", "a\nb\n", "a\r\nb", "a\r\n",
                     "a\r\r\nb", "a\rb", "\r\n", "  \n\t\n", "one\ntwo\nthree",
                     "fn main() {\n    println!(\"hi\");\n}\n", "αβ\nγ"];
        for case in cases {
            let expected = case.lines().collect::<Vec<&str>>();
            let actual = get_lines_of(case).map(|(_, line)| line).collect::<Vec<&str>>();
            assert_eq!(expected, actual, "disagreed on {case:?}");
        }
    }

    // The resolution reads a symbol identity beside every position; the cases here are all the one
    // '/*' '*/' pair, so the helper pins the identity to 0 and the assertions stay bare positions.
    fn resolved_double_counting(start_indices: Vec<usize>, end_indices: Vec<usize>, is_comment_open: bool)
    -> (Vec<usize>, Vec<usize>) {
        let language = Language::new("one-pair", ["x"], build_backslashed_quotes(), ["//"], &[("/*", "*/")], []);
        let mut starts = start_indices.into_iter().map(|x| (x, 0u8, 0u8)).collect::<Vec<_>>();
        let mut ends = end_indices.into_iter().map(|x| (x, 0u8, 0u8)).collect::<Vec<_>>();
        resolve_double_counting_of_adjacent_start_and_end_symbols(&mut starts, &mut ends, is_comment_open, &language);
        (starts.into_iter().map(|(x, _, _)| x).collect(), ends.into_iter().map(|(x, _, _)| x).collect())
    }

    #[test]
    fn a_block_opener_and_closer_sharing_bytes_are_counted_once() {
        // /*Hello*//* world*//*
        assert_eq!((vec![0,9,19],vec![7,17]), resolved_double_counting(vec![0,9,19], vec![7,17], false));
        // /**//**/
        assert_eq!((vec![0,4],vec![2,6]), resolved_double_counting(vec![0,4], vec![2,6], false));
        // /*/**/*/
        assert_eq!((vec![0,2],vec![4,6]), resolved_double_counting(vec![0,2], vec![4,6], false));

        // /* */*
        assert_eq!((vec![0],vec![3]), resolved_double_counting(vec![0,4], vec![3], false));

        // */* /*/
        assert_eq!((vec![1],vec![5]), resolved_double_counting(vec![1,4], vec![0,5], false));
        assert_eq!((vec![4],vec![0]), resolved_double_counting(vec![1,4], vec![0,5], true));

        // /*/*/ */*/ /* */
        assert_eq!((vec![0,7,11],vec![3,14]), resolved_double_counting(vec![0,2,7,11], vec![1,3,6,8,14], false));
        assert_eq!((vec![7,11],vec![1,3,14]), resolved_double_counting(vec![0,2,7,11], vec![1,3,6,8,14], true));

        // /*/*/ */*/
        assert_eq!((vec![0,7],vec![3]), resolved_double_counting(vec![0,2,7], vec![1,3,6,8], false));
        assert_eq!((vec![7],vec![1,3]), resolved_double_counting(vec![0,2,7], vec![1,3,6,8], true));

        // '*/ */*' with a comment open from the line before, which is the case that decides the two
        // conditions in the loop below 'resolve_collision'. They are not mirror images of each other,
        // and the one that looks like a typo is the one that is right: the end symbol at 0 closes the
        // comment, so the '*/' at 3 is a stray in code and the '/*' at 4 is a real opener. Reading the
        // second condition as the mirror of the first discards the opener instead of the stray, and
        // the whole rest of the file is then counted as code.
        assert_eq!((vec![4],vec![0]), resolved_double_counting(vec![4], vec![0,3], true));

        // /* */*/*//*
        assert_eq!((vec![0,6,9],vec![3]), resolved_double_counting(vec![0,4,6,9], vec![3,5,7], false));
        assert_eq!((vec![0,6,9],vec![3]), resolved_double_counting(vec![0,4,6,9], vec![3,5,7], true));
    }

    // Without the declaration the opener wins and the closer that shares its asterisk is dropped,
    // which is right for C's '/*/' and leaves the rest of the file inside a comment here.
    #[test]
    fn a_symbol_the_character_in_front_of_it_cancels_opens_nothing() {
        let vectorish = Language::new("vectorish", ["vec"], build_backslashed_quotes(), ["//"],
                &[("<*", "*>")], [])
                .with_cancelled_symbols(&[("<*", b'['), ("*>", b'<')]);

        let stats = parse_lines_whole("macro rotate(int[<*>] x)\nreturn x;\n", &vectorish);
        assert_eq!((2, 0), (stats.classes.words_in_code, stats.classes.words_in_comment));

        let plain = Language::new("vectorish", ["vec"], build_backslashed_quotes(), ["//"],
                &[("<*", "*>")], []);
        let stats = parse_lines_whole("macro rotate(int[<*>] x)\nreturn x;\n", &plain);
        assert_eq!((1, 1), (stats.classes.words_in_code, stats.classes.words_in_comment));

        let stats = parse_lines_whole("<*\n a comment\n*>\nreturn x;\n", &vectorish);
        assert_eq!((1, 1), (stats.classes.words_in_code, stats.classes.words_in_comment));
        let stats = parse_lines_whole("<*\n the type int[<*>] holds one lane\n a second line\n*>\nreturn x;\n",
                &vectorish);
        assert_eq!((1, 2), (stats.classes.words_in_code, stats.classes.words_in_comment));
    }

    // Lua's pair is 4 bytes on one side and 2 on the other, and one shared collision window sees a
    // collision in ']]--[[' where the two symbols merely touch: the reopening start is discarded
    // and the rest of the file counts as code. On a 7-line file whose honest counts are code=1
    // comments=6, that reads as code=3 comments=2.
    #[test]
    fn a_close_that_touches_a_reopen_is_not_a_collision_when_the_lengths_differ() {
        let lua_like = Language::new("lua-like", ["x"], build_backslashed_quotes(), ["--"], &[("--[[", "]]")], []);
        // ]]--[[ with the block open from the line before: both symbols are real
        let (mut starts, mut ends) = (vec![(2usize, 0u8, 0u8)], vec![(0usize, 0u8, 0u8)]);
        resolve_double_counting_of_adjacent_start_and_end_symbols(&mut starts, &mut ends, true, &lua_like);
        assert_eq!((vec![(2, 0, 0)], vec![(0, 0, 0)]), (starts, ends));

        // and the whole shape read as a line: it closes and reopens, so it ends still open, and
        // whitespace between the two symbols is not code
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("]]--[[", &LUA, Some(0), None));
        assert_eq!(TextInfo::with_open_comment(0),
                bounds_multi("]]  --[[ reopened", &LUA, Some(0), None));
        // an HTML-shaped pair, 4 against 3, through a language declaring no line comments
        let html : Language = Language::new("html-like", ["html"], build_backslashed_quotes(), [""; 0],
                &[("<!--", "-->")], []);
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi("--><!--", &html, Some(0), None));
        assert_eq!(TextInfo::with_open_comment(0),
                bounds_multi("--> <!-- reopened", &html, Some(0), None));
    }

    #[test]
    fn a_block_comment_end_behind_a_line_comment_is_not_a_delimiter() {
        let line = "Hello world!";
        assert_eq!(Vec::<usize>::new(), comment_delimiters_w_multiline(line, &PHP, &[]));
        let line = "//Hello*/ world!";
        assert_eq!(vec![0], comment_delimiters_w_multiline(line, &PHP, &[7]));
        let line = "///*Hello world!";
        assert_eq!(vec![0], comment_delimiters_w_multiline(line, &PHP, &[]));
        let line = "//*//Hello world!";
        assert_eq!(vec![0], comment_delimiters_w_multiline(line, &PHP, &[2]));
        let line = "//*/#Hello world!";
        assert_eq!(vec![0,4], comment_delimiters_w_multiline(line, &PHP, &[2]));
    }
    
    #[test]
    fn python_quotes_are_read_with_pythons_own_rules() {
        let line = String::from("[\"\\\"\\\"\\\"\",\"'''\",\"\\\"\",\"'\",]");
        assert_eq!(TextInfo::new(Some("[,,,,]".to_owned()),true,None,None),bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("\\''\''");
        assert_eq!(TextInfo::new(Some("\\\'".to_owned()),true,None,Some(1u8)), bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::none_all(true), bounds_multi(&line, &PYTHON, None, Some(1u8)));
        let line = String::from("\'\\'\\'\\\''"); 
        assert_eq!(TextInfo::new(None,true,None,None), bounds_multi(&line, &PYTHON, None,None));
        
        let single_str_opt = Some(1u8);
        let double_str_opt = Some(0u8);
        let single_str_li = TextInfo::with_open_symbol(1);
        let double_str_li = TextInfo::with_open_symbol(0);
    
        let line = String::from("Hello world!");
        assert_eq!(TextInfo::from_slice("Hello world!"),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(single_str_li,bounds_multi(&line, &PYTHON, None,single_str_opt));
        
        //testing comments
        let line = String::from("#Hello world!");
        assert_eq!(single_str_li,bounds_multi(&line, &PYTHON, None,single_str_opt));
        let line = String::from("Hello world!#");
        assert_eq!(TextInfo::from_slice("Hello world!"),bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("Hello# world!");
        assert_eq!(TextInfo::from_slice("Hello"),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(single_str_li,bounds_multi(&line, &PYTHON, None,single_str_opt));
        let line = String::from("Hello## world!");
        assert_eq!(TextInfo::from_slice("Hello"),bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("#Hello# world!");
        assert_eq!(single_str_li,bounds_multi(&line, &PYTHON, None,single_str_opt));
        
        //testing strings 
        let line = String::from("\"Hello world!#");
        assert_eq!(double_str_li,bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("\"Hello\" world!");
        assert_eq!(TextInfo::from_slice_w_literal(" world!"),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::new(Some("Hello".to_owned()), true, None, Some(0u8)),bounds_multi(&line, &PYTHON, None,double_str_opt));
        let line = String::from("Hello world!\"");
        assert_eq!(TextInfo::new(Some("Hello world!".to_owned()), true, None, Some(0u8)),bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("\"'Hello'\" world!");
        assert_eq!(TextInfo::from_slice_w_literal(" world!"),bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("'Hello' world!");
        assert_eq!(TextInfo::from_slice_w_literal(" world!"),bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("'\"He'llo'\" world!'");
        assert_eq!(TextInfo::from_slice_w_literal("llo"),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::new(Some("He".to_owned()), true, None, Some(0u8)),bounds_multi(&line, &PYTHON, None,double_str_opt));
        let line = String::from(r#""""Hello""#);
        assert_eq!(TextInfo::new(None, true, None, None), bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::new(Some("Hello".to_owned()), true, None, Some(0u8)), bounds_multi(&line, &PYTHON, None,double_str_opt));
        let line = String::from(r#"['⣯', '⣟"#); 
        assert_eq!(TextInfo::new(Some("[, ".to_owned()),true,None,Some(1u8)), bounds_multi(&line, &PYTHON, None,None));
        
        //test mixed
        let line = String::from("'Hello#' world!'");
        assert_eq!(TextInfo::new(Some(" world!".to_owned()), true, None, Some(1u8)),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::from_slice_w_literal("Hello"),bounds_multi(&line, &PYTHON, None,single_str_opt));
        let line = String::from("'Hello'# world!'");
        assert_eq!(TextInfo::none_all(true),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::from_slice_w_literal("Hello"),bounds_multi(&line, &PYTHON, None,single_str_opt));
        let line = String::from("''#Hello");
        assert_eq!(TextInfo::none_all(true),bounds_multi(&line, &PYTHON, None,None));
        let line = String::from("'''#'''Hello world!'");
        assert_eq!(TextInfo::new(Some("Hello world!".to_owned()), true, None, Some(1u8)),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::none_all(true),bounds_multi(&line, &PYTHON, None,single_str_opt));
        assert_eq!(TextInfo::with_open_symbol(0),bounds_multi(&line, &PYTHON, None,double_str_opt));
        let line = String::from("Hello'###'\"world!\"");
        assert_eq!(TextInfo::from_slice_w_literal("Hello"),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::none_all(true),bounds_multi(&line, &PYTHON, None,single_str_opt));
        assert_eq!(TextInfo::new(Some("world!".to_owned()), true, None, Some(0u8)),bounds_multi(&line, &PYTHON, None,double_str_opt));
        let line = String::from("\"//'''\"Hello'\"world!");
        assert_eq!(TextInfo::new(Some("Hello".to_owned()), true, None, Some(1u8)),bounds_multi(&line, &PYTHON, None,None));
        assert_eq!(TextInfo::from_slice_w_literal("world!"),bounds_multi(&line, &PYTHON, None,single_str_opt));
        assert_eq!(TextInfo::new(Some("//".to_owned()), true, None, Some(0u8)),bounds_multi(&line, &PYTHON, None,double_str_opt));
    }
    
    #[test]
    fn java_strings_and_block_comments_are_read_with_javas_own_rules() {
        let double_str_opt = Some(0u8);

        let line = String::from("Hello world!");
        assert_eq!(TextInfo::with_open_comment(0),bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::with_open_symbol(0),bounds_multi(&line, &JAVA, None, double_str_opt));
        assert_eq!(TextInfo::from_slice("Hello world!"),bounds_multi(&line, &JAVA, None, None));
        
        //testing only multiline comment combinations
        let line = String::from("*/Hello world!");
        assert_eq!(TextInfo::from_slice("Hello world!"),bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::from_slice("*/Hello world!"),bounds_multi(&line, &JAVA, None, None));
        let line = String::from("Hello/* ffd /**//*erer */ world!");
        assert_eq!(TextInfo::from_slice(" world!"),bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::from_slice("Hello world!"),bounds_multi(&line, &JAVA, None, None));
        let line = String::from("Hello*//**//**/ world!");
        assert_eq!(TextInfo::from_slice(" world!"),bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::from_slice("Hello*/ world!"),bounds_multi(&line, &JAVA, None, None));
        let line = String::from("*//*Hello/**/ world!");
        assert_eq!(TextInfo::from_slice(" world!"),bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::from_slice("*/ world!"),bounds_multi(&line, &JAVA, None, None));
        let line = String::from("Hello world*/");
        assert_eq!(TextInfo::none_all(false), bounds_multi(&line, &JAVA, Some(0), None));
        let line = String::from("*/Hello world!/**/");
        assert_eq!(TextInfo::from_slice("Hello world!"), bounds_multi(&line, &JAVA, Some(0), None));
        let line = String::from("Hello world*//**/");
        assert_eq!(TextInfo::none_all(false), bounds_multi(&line, &JAVA, Some(0), None));
        let line = String::from("*/He/**//*llo world*/!/**/");
        assert_eq!(TextInfo::from_slice("He!"), bounds_multi(&line, &JAVA, Some(0), None));
        let line = String::from("Hello world*/!");
        assert_eq!(TextInfo::from_slice("!"), bounds_multi(&line, &JAVA, Some(0), None));
        let line = String::from("/*H*/ello world/*!");
        assert_eq!(TextInfo::new(Some("ello world".to_string()), false, Some((0, 1)), None), bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::new(Some("ello world".to_string()), false, Some((0, 1)), None), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("/*H*/e/*llo world!");
        assert_eq!(TextInfo::new(Some("e".to_string()), false, Some((0, 1)), None), bounds_multi(&line, &JAVA, Some(0), None));
        
        //testing only string symbols
        let line = String::from("\"");
        assert_eq!(TextInfo::with_open_symbol(0), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("\"Hello\"");
        assert_eq!(TextInfo::new(Some("Hello".to_string()), true, None, Some(0u8)), bounds_multi(&line, &JAVA, None, double_str_opt));
        assert_eq!(TextInfo::none_all(true), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("\"\"Hello");
        assert_eq!(TextInfo::with_open_symbol(0), bounds_multi(&line, &JAVA, None, double_str_opt));
        assert_eq!(TextInfo::from_slice_w_literal("Hello"), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("\"\"");
        assert_eq!(TextInfo::with_open_symbol(0), bounds_multi(&line, &JAVA, None, double_str_opt));
        assert_eq!(TextInfo::none_all(true), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("\"\"Hello");
        assert_eq!(TextInfo::from_slice_w_literal("Hello"), bounds_multi(&line, &JAVA, None, None));
        let line  = String::from("Hel\"\"lo");
        assert_eq!(TextInfo::from_slice_w_literal("Hello"), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("\"\"He\"\"\"ll\"o");
        assert_eq!(TextInfo::from_slice_w_literal("Heo"), bounds_multi(&line, &JAVA, None, None));
        let line = String::from(r#""""Hello""#);
        assert_eq!(TextInfo::new(None, true, None, None), bounds_multi(&line, &JAVA, None, None));
        assert_eq!(TextInfo::new(Some("Hello".to_owned()), true, None, Some(0u8)), bounds_multi(&line, &JAVA, None, double_str_opt));
        
        //testing only comments
        let line = String::from("//");
        assert_eq!(TextInfo::none_all(false), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("Hello//");
        assert_eq!(TextInfo::from_slice("Hello"), bounds_multi(&line, &JAVA, None, None));
        assert_eq!(TextInfo::with_open_comment(0), bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::with_open_symbol(0), bounds_multi(&line, &JAVA, None, double_str_opt));
        let line = String::from("//Hello");
        assert_eq!(TextInfo::none_all(false), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("////Hello");
        assert_eq!(TextInfo::none_all(false), bounds_multi(&line, &JAVA, None, None));
        let line = String::from("He//llo//");
        assert_eq!(TextInfo::from_slice("He"), bounds_multi(&line, &JAVA, None, None));
        
        //testing mixed
        let line = String::from("\"\"\"//\"\"\"Hello world!");
        assert_eq!(TextInfo::from_slice_w_literal("Hello world!"),bounds_multi(&line, &JAVA, None, None));
        assert_eq!(TextInfo::none_all(true),bounds_multi(&line, &JAVA, None, double_str_opt));
        let line = String::from("\"\"one\"//\"\"\"Hello world!");
        assert_eq!(TextInfo::from_slice_w_literal("oneHello world!"),bounds_multi(&line, &JAVA, None, None));
        let line = String::from("\"He\"/*l*/lo//fd");
        assert_eq!(TextInfo::from_slice_w_literal("lo"), bounds_multi(&line, &JAVA, None, None));
        assert_eq!(TextInfo::new(Some("He".to_string()), true, None, Some(0u8)), bounds_multi(&line, &JAVA, None, double_str_opt));
        assert_eq!(TextInfo::from_slice("lo"), bounds_multi(&line, &JAVA, Some(0), None));
        let line = String::from("//\"/**/dfd\"");
        assert_eq!(TextInfo::none_all(false), bounds_multi(&line, &JAVA, None, None));
        assert_eq!(TextInfo::new(Some("dfd".to_string()), true, None, Some(0u8)), bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::new(Some("dfd".to_string()), true, None, Some(0u8)), bounds_multi(&line, &JAVA, None, double_str_opt));
        
        let line  = String::from(
            "Hello /* \
            mefm \" */ \" \
            //*/world!"
        );
        assert_eq!(TextInfo::new(Some("Hello ".to_string()), true, None, Some(0u8)), bounds_multi(&line, &JAVA, None, None));
        assert_eq!(TextInfo::with_open_symbol(0), bounds_multi(&line, &JAVA, Some(0), None));
        assert_eq!(TextInfo::new(Some(" */ ".to_string()), true, None, Some(0u8)), bounds_multi(&line, &JAVA, None, double_str_opt));
    }

    const MARKER: &str = "mezura-expect";
    const LANGUAGE_FIELD: &str = "language=";

    fn fixtures_dir() -> std::path::PathBuf {
        Path::new(FIXTURES_DIR).join("lang")
    }

    // Each fixture declares, on its first line and in its own comment syntax, the counts mezura must
    // produce for it. The counts are hand-verified. The header line itself is a comment, so it is
    // included in 'lines' and excluded from 'code'.
    // Naming the language is only for an extension two of them claim, as MATLAB and Objective-C
    // both claim '.m': the lookup would answer with the tie-break rule and the counts would be that
    // rule's rather than the parser's. It comes last on the line because a language name can hold
    // a space.
    fn parse_expectations(first_line: &str) -> Option<(Option<String>, HashMap<String, usize>)> {
        let after_marker = first_line.split_once(MARKER)?.1;
        // A fixture in a language whose comments are blocks carries the closer on the header line,
        // and the closer is where the declarations end rather than a malformed one. Anything else
        // that is not a 'name=count' is a typo and refuses the header, which is the point.
        let after_marker = ["-->", "*/", "*)", "-}", "]]", "}"].iter()
                .fold(after_marker, |text, closer| text.split(closer).next().unwrap_or(text));
        let (counts, language) = match after_marker.split_once(LANGUAGE_FIELD) {
            Some((before, name)) => (before, Some(name.trim().to_owned())),
            None => (after_marker, None)
        };
        let mut expectations = HashMap::new();
        for entry in counts.split_whitespace() {
            let (key, value) = entry.split_once('=')?;
            expectations.insert(key.to_owned(), value.parse::<usize>().ok()?);
        }

        if expectations.is_empty() { None } else { Some((language, expectations)) }
    }

    fn fixture_paths(root: &Path) -> Vec<std::path::PathBuf> {
        let mut paths = std::fs::read_dir(root)
            .unwrap_or_else(|x| panic!("cannot read the fixture directory {}: {x}", root.display()))
            .flatten()
            .map(|entry| entry.path())
            .filter(|path| path.is_file())
            .collect::<Vec<_>>();
        paths.sort();
        paths
    }

    #[test]
    fn language_fixtures_match_their_declared_counts() {
        let root = fixtures_dir();
        // The same lookup a run uses, name before extension, so a fixture called 'Makefile' is
        // resolved the way the program resolves it and not by a rule of this test's own
        let lookup = fixture_lookup();
        // Built-in defaults only: a preference in the machine's config file must not move a count
        let config = EngineConfig::default();

        let mut failures = Vec::new();
        let mut checked = 0;

        for path in fixture_paths(&root) {
            let name = path.file_name().unwrap().to_string_lossy().into_owned();

            let contents = std::fs::read_to_string(&path).unwrap();
            let Some((declared, expected)) = parse_expectations(contents.lines().next().unwrap_or_default()) else {
                failures.push(format!("{name}: the first line must contain a '{MARKER} lines=N code=N ...' header"));
                continue;
            };

            let lang_name = match declared {
                Some(declared) => std::sync::Arc::from(declared.as_str()),
                None => match lookup.of_path_or_shebang(&path) {
                    Some(found) => found,
                    None => {
                        failures.push(format!("{name}: no supported language claims this name, its extension or its first line"));
                        continue;
                    }
                }
            };
            if !LANGUAGE_MAP_REF.contains_key(lang_name.as_ref()) {
                failures.push(format!("{name}: no language is called '{lang_name}'"));
                continue;
            }

            let language = LANGUAGE_MAP_REF.get(lang_name.as_ref()).unwrap();
            let mut buf = Vec::new();
            let stats = match parse_file_whole(&path, lang_name.as_ref(), &mut buf, &config) {
                Ok(stats) => stats,
                Err(x) => {
                    failures.push(format!("{name}: could not be parsed: {x}"));
                    continue;
                }
            };

            let (lines, code, comments) = content_counts(&stats);
            let mut actual = HashMap::from([
                ("lines".to_owned(), lines),
                ("code".to_owned(), code),
                ("comments".to_owned(), comments),
                ("extra".to_owned(), lines - code - comments),
            ]);
            for (index, keyword) in language.keywords.iter().enumerate() {
                actual.insert(keyword.descriptive_name.clone(), stats.keyword_occurences[index]);
            }

            for (key, expected_value) in &expected {
                match actual.get(key) {
                    Some(actual_value) if actual_value == expected_value => (),
                    Some(actual_value) => failures.push(format!("{name} ({lang_name}): {key} expected {expected_value}, got {actual_value}")),
                    None => {
                        let mut known = actual.keys().cloned().collect::<Vec<_>>();
                        known.sort();
                        failures.push(format!("{name} ({lang_name}): '{key}' is not a countable field. Available: {}", known.join(", ")));
                    }
                }
            }

            // A keyword the fixture does not mention must be absent, otherwise a fixture could
            // quietly stop covering a keyword the moment someone forgets to declare it
            for (index, keyword) in language.keywords.iter().enumerate() {
                let occurrences = stats.keyword_occurences[index];
                if occurrences > 0 && !expected.contains_key(&keyword.descriptive_name) {
                    failures.push(format!("{name} ({lang_name}): found {occurrences} '{}' but the header does not declare them",
                            keyword.descriptive_name));
                }
            }

            checked += 1;
        }

        assert!(checked > 0, "no fixtures were checked, is {} populated?", root.display());
        assert!(failures.is_empty(), "\n{} fixture check(s) failed:\n  {}\n", failures.len(), failures.join("\n  "));
    }

    // Counting and explaining are one function, but the explain records are written beside the
    // class counts and a slip would part them. Every fixture goes through both, compared class by
    // class and line by line.
    #[test]
    fn explaining_a_file_answers_exactly_what_counting_it_does() {
        let lookup = fixture_lookup();
        let config = EngineConfig::default();

        let mut checked = 0;
        for path in fixture_paths(&fixtures_dir()) {
            let name = path.file_name().unwrap().to_string_lossy().into_owned();
            if name.ends_with(".md") { continue; }
            let Some(lang_name) = lookup.of_path_or_shebang(&path) else { continue };

            let mut buf = Vec::new();
            let counted = parse_file_report(&path, lang_name.as_ref(), &mut buf, &config)
                    .unwrap_or_else(|x| panic!("{name} could not be counted: {x}"));
            let raw = std::fs::read_to_string(&path)
                    .unwrap_or_else(|x| panic!("{name} could not be read: {x}"));
            let (contents, explained, log) = explain_parsed_file(raw, lang_name.as_ref(), &shipped_lookup(), &config);

            // The spans of every line partition its trimmed text: they start at the first byte of
            // text, touch each other with no gap and no overlap, and stop at the last. A blank
            // line has none.
            for (at, (raw_line, record)) in contents.lines().zip(log.records()).enumerate() {
                let trimmed = raw_line.trim_ascii();
                if trimmed.is_empty() {
                    assert!(record.spans.is_empty(), "{name}:{}: a blank line got spans", at + 1);
                    continue;
                }
                let lead = raw_line.len() - raw_line.trim_ascii_start().len();
                assert_eq!(lead, record.spans[0].from,
                        "{name}:{}: the first span starts past the text", at + 1);
                assert_eq!(lead + trimmed.len(), record.spans.last().unwrap().to,
                        "{name}:{}: the last span stops short of the text", at + 1);
                for pair in record.spans.windows(2) {
                    assert_eq!(pair[0].to, pair[1].from,
                            "{name}:{}: spans leave a gap or overlap", at + 1);
                }
                for span in &record.spans {
                    assert!(span.from < span.to, "{name}:{}: an empty span", at + 1);
                }
            }

            let mut lines_per_language = HashMap::<String, usize>::new();
            for record in log.records() {
                *lines_per_language.entry(log.get_language_name_of(record).to_owned()).or_default() += 1;
            }
            let mut expected = HashMap::<String, usize>::new();
            *expected.entry(lang_name.to_string()).or_default() += counted.shell.lines;
            for section in &counted.sections {
                *expected.entry(section.language.clone()).or_default() += section.stats.lines;
            }
            expected.retain(|_, lines| *lines > 0);
            assert_eq!(expected, lines_per_language, "{name}: lines per language");

            let whole_counted = counted.into_whole();
            let whole_explained = explained.into_whole();
            assert_eq!(whole_counted.classes, whole_explained.classes, "{name}");
            assert_eq!(whole_counted.lines, log.records().len(),
                    "{name}: {} lines got {} records", whole_counted.lines, log.records().len());

            let mut from_records = crate::LineClasses::default();
            for record in log.records() {
                from_records.bump(record.class);
            }
            assert_eq!(whole_counted.classes, from_records,
                    "{name}: the records disagree with the counted classes");
            checked += 1;
        }
        assert!(checked > 30, "only {checked} files were swept");
    }

    #[test]
    fn a_file_is_read_to_its_end_whatever_length_the_listing_gave() {
        let path = std::env::temp_dir().join("a_file_is_read_to_its_end_whatever_length_the_listing_gave.rs");
        std::fs::write(&path, "x".repeat(40)).unwrap();
        let read_with = |size: u64| {
            let mut file = File::open(&path).unwrap();
            read_file_into(&mut file, &mut Vec::new(), size).unwrap()
        };

        assert_eq!(40, read_with(4), "a file longer than the listing said was cut short");
        assert_eq!(40, read_with(0), "a file the listing could not size was cut short");
        assert_eq!(40, read_with(400), "a file shorter than the listing said was read past its end");

        std::fs::remove_file(&path).unwrap();
    }

    #[test]
    fn a_bundle_is_left_out_of_the_counts_and_counted_when_the_flag_asks() {
        let root = std::env::temp_dir().join("mezura_minified_test");
        let _ = std::fs::remove_dir_all(&root);
        std::fs::create_dir_all(&root).unwrap();
        let file = |name: &str, contents: String| {
            let path = root.join(name);
            std::fs::write(&path, contents).unwrap();
            path
        };
        let skipped = |path: &std::path::Path, config: &EngineConfig| {
            let mut buf = Vec::new();
            matches!(parse_file(path, get_size_of(path), "JavaScript", &mut buf, &mut ParseBuffers::default(),
                    &shipped_lookup(), &mut KeywordMatchers::default(),
                    &mut IdentificationMatchers::default(), config, false, None, &HashMap::new()),
                    Ok(FileOutcome::Skipped(ScanSkip::Minified)))
        };

        let payload = format!("var a{};\n", "x".repeat(4000));
        let bundle = file("bundle.js", format!("/*! licence */\n{}", payload.repeat(20)));
        let hand_written = file("app.js", "var a = 1;\nfunction f() { return 2; }\n".repeat(1000));
        // Long lines but too few bytes to matter to any report, so it is never even tested
        let tiny = file("tiny.js", format!("var a{};\n", "x".repeat(5000)));

        let counting_everything = EngineConfig { count_minified: true, ..Default::default() };
        assert!(skipped(&bundle, &EngineConfig::default()), "the bundle was counted");
        assert!(!skipped(&bundle, &counting_everything), "'--count-minified' did not count it");
        assert!(!skipped(&hand_written, &EngineConfig::default()),
                "{} lines of ordinary source were taken for a bundle",
                std::fs::read_to_string(&hand_written).unwrap().lines().count());
        assert!(!skipped(&tiny, &EngineConfig::default()), "a file too small to matter was tested");

        std::fs::remove_dir_all(&root).unwrap();
    }

    #[test]
    fn evidence_matches_at_a_line_start_behind_blanks_and_never_inside_a_word() {
        let language = Language::new("Evid", ["ev"], crate::StringRules::escaping_nothing(), ["//"], &[], [])
                .with_identification(["@property", "class"], ["std::"]);
        let matcher = IdentificationMatcher::build(&language).unwrap();
        let finds = |text: &str| matcher.find_evidence(text.as_bytes()).map(|(_, literal)| literal.to_owned());

        assert_eq!(Some("@property".to_owned()), finds("int x;\n\t  @property int y;\n"));
        assert_eq!(None, finds("int x; @property int y;\n"), "mid-line matched a line-start literal");
        assert_eq!(Some("class".to_owned()), finds("class Foo {\n"));
        assert_eq!(None, finds("classic_t x;\n"), "a literal kept running into the word after it");
        assert_eq!(Some("std::".to_owned()), finds("int a = std::max(1, 2);\n"));
        assert_eq!(None, finds("plain text\n"));
        assert!(IdentificationMatcher::build(&Language::new("Bare", ["b"],
                crate::StringRules::escaping_nothing(), ["//"], &[], [])).is_none());
    }

    #[test]
    fn evidence_past_the_identification_cap_is_not_read() {
        let language = Language::new("Evid", ["ev"], crate::StringRules::escaping_nothing(), ["//"], &[], [])
                .with_identification(["zeddoc"], [""; 0]);
        let languages = crate::languages::keyed_by_name([language]);
        let contenders = [Arc::<str>::from("Evid")];
        let no_shebangs = HashMap::new();
        let mut matchers = IdentificationMatchers::default();
        let mut identify = |buf: &str| identify_language(buf, &contenders, &languages, &no_shebangs,
                &mut matchers).map(|(name, _)| name.to_string());

        let padding = "aaaaaaaa\n".repeat(IDENTIFICATION_BYTES / 9 + 1);
        assert_eq!(None, identify(&format!("{padding}zeddoc\n")));
        assert_eq!(Some("Evid".to_owned()),
                identify(&format!("{}\nzeddoc\n{padding}", &padding[..IDENTIFICATION_BYTES / 2])));
    }

    #[test]
    fn a_marker_matches_as_a_prefix_on_the_first_two_lines_where_identification_would_not() {
        let matcher = IdentificationMatcher::of(&["-keep".to_owned()], &[".o:".to_owned()]).unwrap();

        assert!(matcher.finds_a_marker("-keepnames class * { *; }\n"));
        assert!(matcher.find_evidence("-keepnames class * { *; }\n".as_bytes()).is_none(),
                "identification loosened into prefix matching");

        assert!(matcher.finds_a_marker(&format!("{} main.o: src\nrest\n", "x".repeat(3 * IDENTIFICATION_BYTES))),
                "a marker at the end of one long first line was not read");
        assert!(matcher.finds_a_marker("main.d: src\n\nlibmain.o: src\n"),
                "a marker on the third line, where cargo writes its artifact rule, was not read");
        assert!(!matcher.finds_a_marker(&format!("{}main.o: y\n", "code\n".repeat(NOT_CODE_MARKER_LINES))),
                "a contains marker past the top lines was believed");
        assert!(matcher.finds_a_marker("\u{feff}-keep class x\n"),
                "a byte order mark defeated a line-start marker");

        let contains_word = IdentificationMatcher::of(&[], &["bundle".to_owned()]).unwrap();
        assert!(!contains_word.finds_a_marker("a bundled thing\n"),
                "a contains marker matched a prefix of a longer word");
        assert!(contains_word.finds_a_marker("a bundle of things\n"));
    }

    // The markers as the tools really write them, taken off files found on a whole drive
    #[test]
    fn a_file_whose_head_says_a_tool_wrote_it_is_left_out() {
        let root = std::env::temp_dir().join("mezura_generated_test");
        let _ = std::fs::remove_dir_all(&root);
        std::fs::create_dir_all(&root).unwrap();
        let body = "var a = 1;\nvar b = 2;\n".repeat(40);
        let skipped = |name: &str, head: &str, config: &EngineConfig| {
            let path = root.join(name);
            std::fs::write(&path, format!("{head}{body}")).unwrap();
            let mut buf = Vec::new();
            matches!(parse_file(&path, get_size_of(&path), "JavaScript", &mut buf, &mut ParseBuffers::default(),
                    &shipped_lookup(), &mut KeywordMatchers::default(),
                    &mut IdentificationMatchers::default(), config, false, None, &HashMap::new()),
                    Ok(FileOutcome::Skipped(ScanSkip::Generated)))
        };
        let default = EngineConfig::default();
        let counting_everything = EngineConfig { count_generated: true, ..Default::default() };

        for (name, head) in [("go.js", "// Code generated by protoc-gen-go. DO NOT EDIT.\n"),
                ("csharp.js", "// <auto-generated>\n"),
                ("cml.js", "/* This file is autogenerated by cml-utils 2024-10-04 */\n"),
                ("meta.js", "/* @generated */\n"),
                ("shouty.js", "/* THIS FILE WAS AUTOGENERATED BY jevents.py */\n"),
                // behind a licence header, which is why the window is not one line
                ("licenced.js", &format!("/*\n{}\n*/\n// <auto-generated>\n", " * SPDX-License-Identifier: GPL-2.0\n".repeat(6)))] {
            assert!(skipped(name, head, &default), "'{}' was counted", head.trim());
            assert!(!skipped(name, head, &counting_everything), "'--count-generated' did not count {name}");
        }

        // ordinary source, and the two shapes that must not fire: a sentence using the words, and
        // the generator itself, whose marker sits past the window it prints into its own output
        assert!(!skipped("plain.js", "// a hand written file\n", &default));
        assert!(!skipped("prose.js", "// The table below was generated by a reference implementation\n", &default));
        let deep = format!("{}\nconsole.log('/* do not edit */');\n", "// padding padding padding\n".repeat(30));
        assert!(!skipped("generator.js", &deep, &default), "the marker was found past the window");

        std::fs::remove_dir_all(&root).unwrap();
    }

    // With the conflict rules a real run has, so that a contested extension resolves here to the
    // language it resolves to on somebody's machine. Without them the tiebreak is alphabetical, and
    // a '.pas' file would be counted as Delphi in the corpus and as Pascal everywhere else.
    fn fixture_lookup() -> LanguageLookup {
        let conflicts = crate::languages::parse_shipped_conflict_rules();
        LanguageLookup {
            by_extension: build_language_map_by(ClaimKind::Extension, &LANGUAGE_MAP_REF,
                    &conflicts.by_extension, &HashMap::new()).0,
            by_filename: build_language_map_by(ClaimKind::Filename, &LANGUAGE_MAP_REF,
                    &conflicts.by_filename, &HashMap::new()).0,
            by_shebang: build_language_map_by(ClaimKind::Shebang, &LANGUAGE_MAP_REF,
                    &HashMap::new(), &HashMap::new()).0,
            extension_rules: HashMap::new()
        }
    }

    #[test]
    fn every_fixture_extension_resolves_to_exactly_one_language() {
        use crate::engine::identity::interpreter_spellings;
        // One map over all three kinds of identity, keyed the way the real maps key each kind, so
        // that a language declaring 'sh' as an extension and as a shebang counts as one claimant
        let mut claimants_of = HashMap::<String, Vec<String>>::new();
        for language in LANGUAGE_MAP_REF.values() {
            let mut claim = |identity: String| {
                let claiming = claimants_of.entry(identity).or_default();
                if !claiming.contains(&language.name) {
                    claiming.push(language.name.clone());
                }
            };
            language.extensions.iter().for_each(|x| claim(ClaimKind::Extension.key_of(x)));
            // A fixture named after a whole filename is resolved by that name, so what has to be
            // uncontested is the name and not the extension its spelling happens to end in
            language.filenames.iter().for_each(|x| claim(ClaimKind::Filename.key_of(x)));
            language.shebangs.iter().for_each(|x| claim(ClaimKind::Shebang.key_of(x)));
        }

        for path in fixture_paths(&fixtures_dir()) {
            // One that says what it is has already answered the question this asks
            let contents = std::fs::read_to_string(&path).unwrap_or_default();
            if parse_expectations(contents.lines().next().unwrap_or_default())
                    .is_some_and(|(declared, _)| declared.is_some()) {
                continue;
            }

            let name = path.file_name().and_then(|x| x.to_str()).unwrap_or_default();
            let as_filename = ClaimKind::Filename.key_of(name);
            let identity = if claimants_of.contains_key(&as_filename) {
                as_filename
            } else if let Some(extension) = path.extension().and_then(|x| x.to_str()) {
                ClaimKind::Extension.key_of(extension)
            } else {
                // An extensionless fixture resolves through its first line, by the same
                // spellings the walk tries, most specific first
                let token = crate::engine::identity::find_interpreter(contents.as_bytes())
                        .and_then(|x| std::str::from_utf8(x).ok()).unwrap_or_default();
                interpreter_spellings(token).into_iter()
                        .find(|spelling| claimants_of.contains_key(spelling))
                        .unwrap_or_else(|| token.to_ascii_lowercase())
            };
            let claimants = claimants_of.get(&identity).cloned().unwrap_or_default();
            assert!(claimants.len() == 1, "the fixture identity '{identity}' is claimed by {} languages ({}), so its counts depend on the tie-break rule",
                    claimants.len(), claimants.join(", "));
        }
    }
}