use std::{
collections::HashMap,
error::Error,
fmt,
hash::{BuildHasherDefault, Hasher},
};
use memchr::memrchr;
use crate::{
CompiledAlternative, Pattern, PatternError, PatternOptions, Prefilter, Token, next_separator,
without_leading_dot_slash,
};
#[derive(Debug, Clone, Default)]
pub struct PatternSet {
patterns: Vec<Pattern>,
exact: KeyIndex,
folded: KeyIndex,
always: Vec<usize>,
}
#[derive(Clone, Copy)]
enum Entry {
Whole,
Path,
GlobPath,
}
impl Entry {
fn matches(self, pattern: &Pattern, path: &[u8]) -> bool {
match self {
Self::Whole => pattern.is_match(path),
Self::Path => pattern.is_match_path(path),
Self::GlobPath => pattern.is_match_glob_path(path),
}
}
fn view(self, path: &[u8]) -> &[u8] {
match self {
Self::Whole => path,
Self::Path | Self::GlobPath => without_leading_dot_slash(path),
}
}
}
impl PatternSet {
pub fn new<I>(patterns: I, options: PatternOptions) -> Result<Self, PatternSetError>
where
I: IntoIterator,
I::Item: AsRef<[u8]>,
{
let patterns = patterns
.into_iter()
.enumerate()
.map(|(index, pattern)| {
Pattern::compile(pattern, options).map_err(|error| PatternSetError { index, error })
})
.collect::<Result<Vec<_>, _>>()?;
Ok(Self::from_patterns(patterns))
}
fn from_patterns(patterns: Vec<Pattern>) -> Self {
let mut set = Self {
patterns,
..Self::default()
};
for (index, pattern) in set.patterns.iter().enumerate() {
let keys = pattern
.alternatives
.iter()
.map(Key::of)
.collect::<Option<Vec<_>>>();
let Some(keys) = keys else {
set.always.push(index);
continue;
};
let target = if pattern.options.case_insensitive {
&mut set.folded
} else {
&mut set.exact
};
for key in keys {
target.insert(key, index, pattern.options.case_insensitive);
}
}
set
}
#[must_use]
pub fn len(&self) -> usize {
self.patterns.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.patterns.is_empty()
}
#[must_use]
pub fn is_match(&self, path: impl AsRef<[u8]>) -> bool {
self.any(path.as_ref(), Entry::Whole)
}
#[must_use]
pub fn is_match_path(&self, path: impl AsRef<[u8]>) -> bool {
self.any(path.as_ref(), Entry::Path)
}
#[must_use]
pub fn is_match_glob_path(&self, path: impl AsRef<[u8]>) -> bool {
self.any(path.as_ref(), Entry::GlobPath)
}
pub fn matches_into(&self, path: impl AsRef<[u8]>, matches: &mut Vec<usize>) {
self.collect(path.as_ref(), Entry::Whole, matches);
}
pub fn matches_path_into(&self, path: impl AsRef<[u8]>, matches: &mut Vec<usize>) {
self.collect(path.as_ref(), Entry::Path, matches);
}
pub fn matches_glob_path_into(&self, path: impl AsRef<[u8]>, matches: &mut Vec<usize>) {
self.collect(path.as_ref(), Entry::GlobPath, matches);
}
fn any(&self, path: &[u8], entry: Entry) -> bool {
let found = self.for_each_candidate(entry.view(path), |index| {
entry.matches(&self.patterns[index], path)
});
found
|| self
.always
.iter()
.any(|&index| entry.matches(&self.patterns[index], path))
}
fn collect(&self, path: &[u8], entry: Entry, matches: &mut Vec<usize>) {
matches.clear();
self.for_each_candidate(entry.view(path), |index| {
matches.push(index);
false
});
if !matches.is_empty() {
matches.extend_from_slice(&self.always);
matches.sort_unstable();
matches.dedup();
} else {
matches.extend_from_slice(&self.always);
}
matches.retain(|&index| entry.matches(&self.patterns[index], path));
}
fn for_each_candidate(&self, view: &[u8], mut visit: impl FnMut(usize) -> bool) -> bool {
if self.exact.find(view, &mut visit) {
return true;
}
if self.folded.is_empty() {
return false;
}
if view.iter().any(u8::is_ascii_uppercase) {
let mut buffer = [0; 256];
if let Some(folded) = buffer.get_mut(..view.len()) {
folded.copy_from_slice(view);
folded.make_ascii_lowercase();
self.folded.find(folded, &mut visit)
} else {
self.folded.find(&view.to_ascii_lowercase(), &mut visit)
}
} else {
self.folded.find(view, &mut visit)
}
}
}
impl FromIterator<Pattern> for PatternSet {
fn from_iter<I: IntoIterator<Item = Pattern>>(patterns: I) -> Self {
Self::from_patterns(patterns.into_iter().collect())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PatternSetError {
index: usize,
error: PatternError,
}
impl PatternSetError {
#[must_use]
pub const fn index(&self) -> usize {
self.index
}
#[must_use]
pub const fn pattern_error(&self) -> &PatternError {
&self.error
}
}
impl fmt::Display for PatternSetError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "pattern {}: {}", self.index, self.error)
}
}
impl Error for PatternSetError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
Some(&self.error)
}
}
enum Key {
Extension(Vec<u8>),
Component(Vec<u8>),
Suffix(Vec<u8>),
Prefix(Vec<u8>),
}
impl Key {
fn of(alternative: &CompiledAlternative) -> Option<Self> {
if alternative.extglob.is_some() {
return None;
}
let stripped = alternative.raw.starts_with(b"./")
&& matches!(
alternative.tokens.as_slice(),
[Token::Literal(dot), Token::Separator, ..] if dot == b"."
);
let tokens = if stripped {
&alternative.tokens[2..]
} else {
&alternative.tokens[..]
};
let fixed = Prefilter::compile(tokens);
let suffix = last_component(&fixed.suffix);
if let Some(name) = tokens
.len()
.checked_sub(1)
.and_then(|last| whole_component_at(tokens, last))
{
return Some(Self::Component(name.to_vec()));
}
if let Some(dot) = memrchr(b'.', suffix) {
return Some(Self::Extension(suffix[dot..].to_vec()));
}
if let Some(component) = (0..tokens.len())
.rev()
.filter_map(|index| whole_component_at(tokens, index))
.max_by_key(|literal| literal.len())
{
return Some(Self::Component(component.to_vec()));
}
if !suffix.is_empty() {
return Some(Self::Suffix(suffix.to_vec()));
}
let prefix = first_component(&fixed.prefix);
(!stripped && !prefix.is_empty()).then(|| Self::Prefix(prefix.to_vec()))
}
}
fn last_component(run: &[u8]) -> &[u8] {
run.iter()
.rposition(|&byte| matches!(byte, b'/' | b'\\'))
.map_or(run, |separator| &run[separator + 1..])
}
fn first_component(run: &[u8]) -> &[u8] {
run.iter()
.position(|&byte| matches!(byte, b'/' | b'\\'))
.map_or(run, |separator| &run[..separator])
}
fn whole_component_at(tokens: &[Token], index: usize) -> Option<&[u8]> {
let Token::Literal(literal) = &tokens[index] else {
return None;
};
let starts =
index == 0 || matches!(tokens[index - 1], Token::Separator | Token::RecursivePrefix);
let ends = matches!(tokens.get(index + 1), None | Some(Token::Separator));
(starts
&& ends
&& !literal.is_empty()
&& !literal.iter().any(|&byte| matches!(byte, b'/' | b'\\')))
.then_some(literal.as_slice())
}
#[derive(Debug, Clone, Default)]
struct KeyIndex {
extensions: KeyMap,
components: KeyMap,
suffixes: Vec<(usize, KeyMap)>,
prefixes: Vec<(usize, KeyMap)>,
}
#[derive(Debug, Clone, Default)]
struct KeyMap {
members: HashMap<Box<[u8]>, Vec<usize>, BuildHasherDefault<KeyHasher>>,
lengths: u64,
}
impl KeyMap {
fn length_bit(length: usize) -> u64 {
1 << length.min(63)
}
fn is_empty(&self) -> bool {
self.members.is_empty()
}
#[cfg(test)]
fn len(&self) -> usize {
self.members.len()
}
fn insert(&mut self, key: Vec<u8>, index: usize) {
self.lengths |= Self::length_bit(key.len());
let members = self.members.entry(key.into_boxed_slice()).or_default();
if members.last() != Some(&index) {
members.push(index);
}
}
fn get(&self, key: &[u8]) -> Option<&[usize]> {
if self.lengths & Self::length_bit(key.len()) == 0 {
return None;
}
self.members.get(key).map(Vec::as_slice)
}
}
impl KeyIndex {
fn is_empty(&self) -> bool {
self.extensions.is_empty()
&& self.components.is_empty()
&& self.suffixes.is_empty()
&& self.prefixes.is_empty()
}
fn insert(&mut self, key: Key, index: usize, fold: bool) {
let (map, mut bytes) = match key {
Key::Extension(bytes) => (&mut self.extensions, bytes),
Key::Component(bytes) => (&mut self.components, bytes),
Key::Suffix(bytes) => (by_length(&mut self.suffixes, bytes.len()), bytes),
Key::Prefix(bytes) => (by_length(&mut self.prefixes, bytes.len()), bytes),
};
if fold {
bytes.make_ascii_lowercase();
}
map.insert(bytes, index);
}
fn find(&self, view: &[u8], visit: &mut impl FnMut(usize) -> bool) -> bool {
let mut hits = |map: &KeyMap, key: &[u8]| {
map.get(key)
.is_some_and(|members| members.iter().any(|&index| visit(index)))
};
if !self.extensions.is_empty()
&& let Some(dot) = memrchr(b'.', view)
&& hits(&self.extensions, &view[dot..])
{
return true;
}
if !self.components.is_empty() {
let mut rest = view;
loop {
let end = next_separator(rest).unwrap_or(rest.len());
if hits(&self.components, &rest[..end]) {
return true;
}
if end == rest.len() {
break;
}
rest = &rest[end + 1..];
}
}
for (length, map) in &self.suffixes {
if let Some(start) = view.len().checked_sub(*length)
&& hits(map, &view[start..])
{
return true;
}
}
for (length, map) in &self.prefixes {
if let Some(prefix) = view.get(..*length)
&& hits(map, prefix)
{
return true;
}
}
false
}
}
fn by_length(maps: &mut Vec<(usize, KeyMap)>, length: usize) -> &mut KeyMap {
let position = match maps.iter().position(|(existing, _)| *existing == length) {
Some(position) => position,
None => {
maps.push((length, KeyMap::default()));
maps.len() - 1
}
};
&mut maps[position].1
}
#[derive(Debug, Clone, Copy, Default)]
struct KeyHasher(u64);
impl KeyHasher {
const SEED: u64 = 0x51_7c_c1_b7_27_22_0a_95;
fn mix(&mut self, word: u64) {
self.0 = (self.0.rotate_left(5) ^ word).wrapping_mul(Self::SEED);
}
}
impl Hasher for KeyHasher {
fn write(&mut self, bytes: &[u8]) {
let (words, rest) = bytes.as_chunks::<8>();
for &word in words {
self.mix(u64::from_le_bytes(word));
}
if !rest.is_empty() {
let mut word = [0; 8];
word[..rest.len()].copy_from_slice(rest);
self.mix(u64::from_le_bytes(word));
}
}
fn write_usize(&mut self, value: usize) {
self.mix(value as u64);
}
fn finish(&self) -> u64 {
self.0
}
}
#[cfg(test)]
mod tests {
use super::*;
fn member_verdicts(patterns: &[Pattern], path: &[u8]) -> [Vec<usize>; 3] {
let indices = |entry: Entry| {
patterns
.iter()
.enumerate()
.filter_map(|(index, pattern)| entry.matches(pattern, path).then_some(index))
.collect::<Vec<_>>()
};
[
indices(Entry::Whole),
indices(Entry::Path),
indices(Entry::GlobPath),
]
}
fn assert_set_agrees(
set: &PatternSet,
patterns: &[Pattern],
path: &[u8],
shown: &dyn Fn() -> String,
) {
let [whole, path_filter, glob_path] = member_verdicts(patterns, path);
let mut matches = vec![usize::MAX];
set.matches_into(path, &mut matches);
assert_eq!(matches, whole, "matches_into: {}", shown());
set.matches_path_into(path, &mut matches);
assert_eq!(matches, path_filter, "matches_path_into: {}", shown());
set.matches_glob_path_into(path, &mut matches);
assert_eq!(matches, glob_path, "matches_glob_path_into: {}", shown());
assert_eq!(
set.is_match(path),
!whole.is_empty(),
"is_match: {}",
shown()
);
assert_eq!(
set.is_match_path(path),
!path_filter.is_empty(),
"is_match_path: {}",
shown()
);
assert_eq!(
set.is_match_glob_path(path),
!glob_path.is_empty(),
"is_match_glob_path: {}",
shown()
);
}
#[test]
fn set_answers_like_its_members() {
let fragments: &[&[u8]] = &[
b"a",
b"B",
b"ab",
b"node_modules",
b".rs",
b".RS",
b".d.ts",
b".",
b"/",
b"./",
b"*",
b"?",
b"**",
b"**/",
b"/**",
b"[ab]",
b"[!.]",
b"{a,b}",
b"{./a,.rs}",
b"{a/b,x.rs}",
b"\\/",
b"\\.",
b"\\\\",
b"\\*",
b"@(ab|B)",
b"!(nm)",
b"+(a)",
];
let path_pieces: &[&[u8]] = &[
b"a",
b"b",
b"B",
b"ab",
b"AB",
b".",
b"/",
b"./",
b"a/b",
b".rs",
b".RS",
b"x.d.ts",
b"node_modules",
b"NODE_MODULES",
b"\\",
b"",
];
let mut seed = 0x2545_F491_4F6C_DD1D_u64;
let mut next = move |bound: usize| {
seed = seed.wrapping_mul(0x2545_F491_4F6C_DD1D).wrapping_add(1);
(usize::try_from(seed >> 33).expect("31 bits fit a usize")) % bound
};
let mut keyed = 0_usize;
let mut always = 0_usize;
for _ in 0..1_500 {
let mut sources = Vec::new();
let mut patterns = Vec::new();
for _ in 0..1 + next(8) {
let mut source = Vec::new();
for _ in 0..1 + next(5) {
source.extend_from_slice(fragments[next(fragments.len())]);
}
let options = PatternOptions::default()
.braces(next(3) != 0)
.recursive_double_star(next(4) != 0)
.extglob(next(3) == 0)
.match_hidden(next(2) == 0)
.case_insensitive(next(3) == 0)
.escape(next(4) != 0);
if let Ok(pattern) = Pattern::compile(&source, options) {
sources.push((source, options));
patterns.push(pattern);
}
}
let set: PatternSet = patterns.iter().cloned().collect();
keyed += set.len() - set.always.len();
always += set.always.len();
for _ in 0..24 {
let mut path = Vec::new();
for _ in 0..next(7) {
path.extend_from_slice(path_pieces[next(path_pieces.len())]);
}
let shown = || {
format!(
"{:?} against {:?}",
sources
.iter()
.map(|(source, options)| format!(
"{:?} {options:?}",
String::from_utf8_lossy(source)
))
.collect::<Vec<_>>(),
String::from_utf8_lossy(&path)
)
};
assert_set_agrees(&set, &patterns, &path, &shown);
}
}
assert!(
keyed > 2 * always,
"only {keyed} of {} generated members were keyed",
keyed + always
);
}
#[test]
fn realistic_lists_answer_like_their_members() {
let globs = [
"**/*.rs",
"**/*.d.ts",
"src/**/*.{ts,tsx}",
"**/node_modules/**",
"target/**",
"./docs/**/*.md",
"**/Makefile",
"**/*rc",
"test_*",
"**/.git/**",
"*.lock",
"**/*.Snap",
"**/!(index).js",
"src/*/mod.rs",
"**",
"{./a,b}/**",
];
let paths = [
"src/lib.rs",
"./src/lib.rs",
"src/app/view.tsx",
"types/index.d.ts",
"web/node_modules/pkg/index.js",
"target/debug/build",
"target",
"docs/guide/intro.md",
"./docs/intro.md",
"tools/Makefile",
"home/.bashrc",
"test_parser",
".git/HEAD",
"Cargo.lock",
"snapshots/view.snap",
"lib/index.js",
"lib/main.js",
"src/parser/mod.rs",
"a/x",
"./a/x",
"b",
"",
".",
"./",
];
for options in [
PatternOptions::walker(),
PatternOptions::walker().match_hidden(true),
PatternOptions::walker().case_insensitive(true),
PatternOptions::default(),
] {
let set = PatternSet::new(globs, options).expect("the globs are valid");
let patterns = globs
.iter()
.map(|glob| Pattern::compile(glob, options).expect("the glob is valid"))
.collect::<Vec<_>>();
for path in paths {
for path in [path.to_owned(), path.to_ascii_uppercase()] {
let shown = || format!("{path:?} under {options:?}");
assert_set_agrees(&set, &patterns, path.as_bytes(), &shown);
}
}
}
}
#[test]
fn common_shapes_are_keyed() {
let set = PatternSet::new(
[
"**/*.rs",
"**/node_modules/**",
"dist/**",
"**/Makefile",
"**/*rc",
"test_*",
"./src/**",
"*",
"**/*.@(js|ts)",
],
PatternOptions::walker(),
)
.expect("the globs are valid");
assert_eq!(set.exact.extensions.len(), 1);
assert_eq!(set.exact.components.len(), 4);
assert_eq!(set.exact.suffixes.len(), 1);
assert_eq!(set.exact.prefixes.len(), 1);
assert_eq!(set.always, [7, 8]);
}
#[test]
fn compile_errors_name_the_failed_glob() {
let error = PatternSet::new(["*.rs", "*.md", "[a-"], PatternOptions::walker())
.expect_err("the class is never closed");
assert_eq!(error.index(), 2);
assert_eq!(
error.pattern_error(),
&Pattern::compile("[a-", PatternOptions::walker()).expect_err("same error")
);
assert!(error.to_string().starts_with("pattern 2: "));
assert!(
error
.source()
.is_some_and(|source| source.to_string() == error.pattern_error().to_string())
);
}
#[test]
fn pattern_set_is_send_sync_clone_debug() {
fn assert_traits<T: Send + Sync + Clone + fmt::Debug + Default>() {}
assert_traits::<PatternSet>();
fn assert_error<T: Send + Sync + Error + Clone + Eq + 'static>() {}
assert_error::<PatternSetError>();
}
}