use std::cell::Cell;
use std::clone::Clone;
use std::collections::HashMap;
use std::env::current_dir;
use std::ffi::OsStr;
use std::fs::{File, read_dir};
use std::io::{Error, Read};
use std::path::{Component, Path, PathBuf};
use crate::error::{ErrorExt, PreprocessParseErrorExt, printable};
use crate::warning::{Warning, WarningSink};
peg::parser! {
pub(crate) grammar preprocess_grammar(deep: &Cell<bool>, file_end: bool) for str {
rule newline() = "\r\n" / "\n"
rule name() -> String = n:$(['a'..='z' | 'A'..='Z' | '0'..='9' | '_']+) {
n.to_string()
}
rule include_path() -> String =
"\"" path:$((!['"'] [_])*) "\"" { path.to_string() } /
"<" path:$((!['>'] [_])*) ">" { path.to_string() }
rule parameters() -> Vec<String> = "(" [' ' | '\t']* p:(name() ** ([' ' | '\t']* "," [' ' | '\t']*)) [' ' | '\t']* ")" {
p
}
rule definition_value() -> Vec<Token> = ([' ' | '\t']+ / &("\\" newline())) v:(token()*) {
v
}
rule definition() -> Definition = n:name() p:parameters()? v:definition_value()? {
Definition {
name: n,
parameters: p,
value: v.unwrap_or_default(),
local: false
}
}
rule directive() -> Directive =
"#" [' ' | '\t']* "include" [' ' | '\t']+ path:include_path() { Directive::IncludeDirective(path) } /
"#" [' ' | '\t']* "define" [' ' | '\t']+ d:definition() { Directive::DefineDirective(d) } /
"#" [' ' | '\t']* "undef" [' ' | '\t']+ n:name() { Directive::UndefDirective(n) } /
"#" [' ' | '\t']* "ifdef" [' ' | '\t']+ n:name() { Directive::IfDefDirective(n) } /
"#" [' ' | '\t']* "ifndef" [' ' | '\t']+ n:name() { Directive::IfNDefDirective(n) } /
"#" [' ' | '\t']* "else" { Directive::ElseDirective } /
"#" [' ' | '\t']* "endif" { Directive::EndIfDirective }
rule nesting_ok(d: usize) = {?
if deep.get() || d >= MAX_NESTING_DEPTH {
deep.set(true);
Err("parentheses nested within the limit")
} else {
Ok(())
}
}
rule arg_rec(d: usize) = "(" nesting_ok(d) (arg_rec(d + 1) / "\\\\" / ("\\" newline()) / !['\r' | '\n' | '(' | ')'] [_])* ")"
rule argument() -> String = a:$((arg_rec(0) / "\\\\" / ("\\" newline()) / !['\r' | '\n' | ',' | ')'] [_])*) {
a.to_string()
}
pub rule arguments() -> Vec<String> = [' ' | '\t']* "(" [' ' | '\t']* a:(argument() ** ([' ' | '\t']* "," [' ' | '\t']*)) [' ' | '\t']* ")" {
a
}
rule nonmacro_token() -> String = s:$((!macro_proper() !comment_token() !concat_token() ("\\\\" / ("\\" newline()) / !['"' | '\r' | '\n'] [_]))+) {
s.to_string()
}
rule string_token() -> (String, u32) =
s:$("\"" ("\\\\" / ("\\" newline()) / "\"\"" / !['\r' | '\n' | '"'] [_])* "\"") {
let newlines =
u32::try_from(s.chars().filter(|c| c == &'\n').count()).unwrap_or(u32::MAX);
(String::from(s).replace("\r\n", "\n").replace("\\\n", ""), newlines)
}
pub rule macro_proper() -> Macro = quoted:"#"? n:name() args:arguments()? {
Macro {
name: n,
arguments: args,
original: String::new(),
quoted: quoted.is_some()
}
}
rule macro_token() -> Macro = original:$(macro_proper()) {?
parse_macro(original).map_err(|_| "a parseable macro invocation")
}
rule concat_token() = "##"
rule sl_comment() -> u32 = "//" (!newline() [_])* &(newline() / ![_]) {
0
}
rule ml_comment() -> u32 = [' ' | '\t']* "/*" content:$((!"*/" (newline() / [_]))*) "*/" {
u32::try_from(content.chars().filter(|c| c == &'\n').count()).unwrap_or(u32::MAX)
}
rule open_comment() -> u32 = [' ' | '\t']* "/*" content:$([_]*) ![_] {?
if file_end {
let content = content.strip_suffix('\n').unwrap_or(content);
Ok(u32::try_from(content.chars().filter(|c| c == &'\n').count()).unwrap_or(u32::MAX))
} else {
Err("a terminated comment")
}
}
rule comment_token() -> u32 = sl_comment() / ml_comment()
rule token() -> Token =
c:comment_token() { Token::CommentToken(c) } /
c:open_comment() { Token::OpenCommentToken(c) } /
sn:string_token() { Token::NewlineToken(sn.0, sn.1) } /
concat_token() { Token::ConcatToken } /
m:macro_token() { Token::MacroToken(m) } /
nm:nonmacro_token() { Token::RegularToken(nm) }
pub rule tokens() -> Vec<Token> = t:(token()*) {
t
}
rule line() -> Line =
[' ' | '\t']* d:directive() cmts:(([' ' | '\t']* c:comment_token() { c })*) open:([' ' | '\t']* c:open_comment() { c })? [' ' | '\t']* {
Line::DirectiveLine(d, cmts.iter().sum::<u32>().saturating_add(open.unwrap_or(0)), open.is_some())
} /
[' ' | '\t']* t:tokens() { Line::TokenLine(t) }
pub rule file() -> Vec<Line> = lines:(line() ** (newline())) ![_] {
lines
}
}
}
#[derive(Clone, Debug)]
pub(crate) struct Definition {
name: String,
parameters: Option<Vec<String>>,
value: Vec<Token>,
local: bool,
}
#[derive(Debug)]
#[expect(
clippy::enum_variant_names,
reason = "the variants are named after the directives they stand for"
)]
pub(crate) enum Directive {
IncludeDirective(String),
DefineDirective(Definition),
UndefDirective(String),
IfDefDirective(String),
IfNDefDirective(String),
ElseDirective,
EndIfDirective,
}
#[derive(Debug)]
pub(crate) struct Macro {
name: String,
arguments: Option<Vec<String>>,
original: String,
quoted: bool,
}
#[derive(Debug)]
#[expect(
clippy::enum_variant_names,
reason = "the variants are named after the tokens they stand for"
)]
pub(crate) enum Token {
RegularToken(String),
NewlineToken(String, u32),
MacroToken(Macro),
CommentToken(u32),
OpenCommentToken(u32),
ConcatToken,
}
#[derive(Debug)]
pub(crate) enum Line {
DirectiveLine(Directive, u32, bool),
TokenLine(Vec<Token>),
}
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct IncludeFolders {
pub search: Vec<PathBuf>,
pub direct: Vec<PathBuf>,
pub memory: Option<MemoryIncludes>,
}
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct MemoryIncludes {
prefix: String,
files: std::collections::BTreeMap<String, (String, Vec<u8>)>,
unreadable: std::collections::BTreeMap<String, String>,
}
impl MemoryIncludes {
#[must_use]
pub fn new(prefix: &str, files: Vec<(String, Vec<u8>)>) -> Self {
let mut map = std::collections::BTreeMap::new();
for (name, content) in files {
map.entry(name.replace('\\', "/").to_lowercase())
.or_insert((name, content));
}
MemoryIncludes {
prefix: prefix.replace('\\', "/").trim_matches('/').to_lowercase(),
files: map,
unreadable: std::collections::BTreeMap::new(),
}
}
#[must_use]
pub fn with_unreadable(mut self, entries: Vec<(String, String)>) -> Self {
for (name, reason) in entries {
self.unreadable
.entry(name.replace('\\', "/").to_lowercase())
.or_insert(reason);
}
self
}
#[must_use]
pub fn get(&self, name: &str) -> Option<&[u8]> {
self.files
.get(&name.replace('\\', "/").to_lowercase())
.map(|(_, content)| content.as_slice())
}
fn find(
&self,
include_path: &str,
origin: Option<&Path>,
line: u32,
) -> Result<(&str, &[u8]), Error> {
let bad = |why: &str| {
error!(
"Include \"{}\" is not allowed ({why}); only files of the same PBO can be included.",
printable(include_path)
)
};
let key = if let Some(rest) = include_path.strip_prefix('\\') {
let rest = rest.replace('\\', "/").to_lowercase();
let below = if self.prefix.is_empty() {
None
} else {
rest.strip_prefix(&self.prefix)
.and_then(|r| r.strip_prefix('/'))
};
let Some(below) = below else {
return Err(include_not_found(include_path, origin, line));
};
if crate::pbo::safe_relative_path(below).is_none() {
return Err(bad("invalid path"));
}
below.to_string()
} else {
let folder = origin
.map(|o| o.to_string_lossy().replace('\\', "/").to_lowercase())
.and_then(|o| o.rsplit_once('/').map(|(f, _)| f.to_string()));
let mut parts: Vec<String> = folder
.iter()
.flat_map(|f| f.split('/'))
.map(str::to_string)
.collect();
if include_path.starts_with('/') {
return Err(bad("absolute, drive or invalid path"));
}
for component in include_path.replace('\\', "/").to_lowercase().split('/') {
if component == ".." {
if parts.pop().is_none() {
return Err(bad("`..` leaves the PBO"));
}
} else if crate::pbo::safe_relative_path(component).is_some() {
parts.push(component.to_string());
} else {
return Err(bad("absolute, drive or invalid path"));
}
}
parts.join("/")
};
if let Some((name, content)) = self.files.get(&key) {
return Ok((name.as_str(), content.as_slice()));
}
if let Some(reason) = self.unreadable.get(&key) {
return Err(match origin {
Some(o) => error!(
"Include \"{}\" (from \"{}:{line}\") cannot be read: {reason}",
printable(include_path),
printable(&o.display().to_string())
),
None => error!(
"Include \"{}\" (line {line}) cannot be read: {reason}",
printable(include_path)
),
});
}
Err(include_not_found(include_path, origin, line))
}
}
impl IncludeFolders {
#[must_use]
pub fn new(search: Vec<PathBuf>) -> Self {
IncludeFolders {
search,
direct: Vec::new(),
memory: None,
}
}
#[must_use]
pub fn with_memory(mut self, memory: MemoryIncludes) -> Self {
self.memory = Some(memory);
self
}
#[must_use]
pub fn with_direct(mut self, direct: Vec<PathBuf>) -> Self {
self.direct = direct;
self
}
}
#[derive(Debug)]
pub struct PreprocessInfo {
pub(crate) line_origins: Vec<(u32, Option<PathBuf>)>,
import_stack: Vec<PathBuf>,
include_depth: usize,
expanded: usize,
expanded_bytes: usize,
steps: usize,
includes: usize,
include_bytes: usize,
pub(crate) ends_in_entry: bool,
}
impl PreprocessInfo {
#[must_use]
pub fn line_origins(&self) -> &[(u32, Option<PathBuf>)] {
&self.line_origins
}
}
pub(crate) const MAX_NESTING_DEPTH: usize = 128;
pub(crate) fn nesting_error() -> Error {
error!("Nesting is deeper than {MAX_NESTING_DEPTH} levels")
}
fn parse_checked<T, E>(
parse: impl FnOnce(&Cell<bool>) -> Result<T, E>,
on_error: impl FnOnce(E) -> Error,
) -> Result<T, Error> {
let deep = Cell::new(false);
let result = parse(&deep);
if deep.get() {
return Err(nesting_error());
}
result.map_err(on_error)
}
const MAX_EXPANSION_DEPTH: usize = 64;
const MAX_LINE_EXPANSION: usize = 500_000;
const MAX_FILE_EXPANSION: usize = 16_000_000;
const MAX_LINE_BYTES: usize = 64 << 20;
const MAX_FILE_BYTES: usize = 512 << 20;
const MAX_LINE_STEPS: usize = 100_000;
const MAX_FILE_STEPS: usize = 100_000;
struct Budget {
tokens: usize,
bytes: usize,
steps: usize,
file_steps: usize,
}
impl Budget {
fn new(file_steps: usize) -> Budget {
Budget {
tokens: 0,
bytes: 0,
steps: 0,
file_steps,
}
}
fn step(&mut self, name: &str) -> Result<(), Error> {
self.steps = self.steps.saturating_add(1);
if self.steps > MAX_LINE_STEPS {
return Err(error!(
"Macro \"{}\" expands to more than {MAX_LINE_STEPS} macro calls in one line",
printable(name)
));
}
if self.file_steps.saturating_add(self.steps) > MAX_FILE_STEPS {
return Err(error!(
"The file and its includes reach more than {MAX_FILE_STEPS} macro calls (at macro \"{}\")",
printable(name)
));
}
self.spend(1, name.len().max(1), name)
}
fn spend(&mut self, tokens: usize, bytes: usize, name: &str) -> Result<(), Error> {
self.tokens = self.tokens.saturating_add(tokens);
self.bytes = self.bytes.saturating_add(bytes);
if self.tokens > MAX_LINE_EXPANSION || self.bytes > MAX_LINE_BYTES {
return Err(error!(
"Macro \"{}\" expands to more than {MAX_LINE_EXPANSION} tokens or {MAX_LINE_BYTES} bytes in one line",
printable(name)
));
}
Ok(())
}
}
fn token_bytes(token: &Token) -> usize {
match token {
Token::RegularToken(s) | Token::NewlineToken(s, _) => s.len().max(1),
Token::MacroToken(m) => m.original.len().max(1),
Token::CommentToken(_) | Token::OpenCommentToken(_) | Token::ConcatToken => 1,
}
}
const MAX_INCLUDE_DEPTH: usize = 32;
const MAX_INCLUDES: usize = 10_000;
const MAX_INCLUDE_BYTES: usize = 64 << 20;
fn identity(path: &Path, memory: bool) -> PathBuf {
if memory {
return PathBuf::from(path.to_string_lossy().replace('\\', "/").to_lowercase());
}
path.canonicalize().unwrap_or_else(|_| path.to_path_buf())
}
fn parse_macro(input: &str) -> Result<Macro, Error> {
let without_original = parse_checked(
|deep| preprocess_grammar::macro_proper(input, deep, false),
|e| error!("Failed to parse macro invocation \"{input}\": {e}"),
)?;
Ok(Macro {
original: String::from(input),
..without_original
})
}
impl Clone for Macro {
fn clone(&self) -> Macro {
Macro {
name: self.name.clone(),
arguments: self.arguments.clone(),
original: self.original.clone(),
quoted: self.quoted,
}
}
}
impl Clone for Token {
fn clone(&self) -> Token {
match self {
Token::RegularToken(s) => Token::RegularToken(s.clone()),
Token::NewlineToken(s, n) => Token::NewlineToken(s.clone(), *n),
Token::MacroToken(m) => Token::MacroToken(m.clone()),
Token::CommentToken(n) => Token::CommentToken(*n),
Token::OpenCommentToken(n) => Token::OpenCommentToken(*n),
Token::ConcatToken => Token::ConcatToken,
}
}
}
impl Definition {
fn value(
&self,
arguments: Option<&[String]>,
def_map: &HashMap<String, Definition>,
stack: &[Definition],
depth: usize,
mismatches: &mut Vec<String>,
budget: &mut Budget,
) -> Result<Option<Vec<Token>>, Error> {
let params = self.parameters.clone().unwrap_or_default();
let args = arguments.unwrap_or(&[]);
let empty_call = params.is_empty()
&& args.len() == 1
&& args.first().is_some_and(|a| a.trim().is_empty());
if params.len() != args.len() && !empty_call {
return Ok(None);
}
let mut tokens = self.value.clone();
if stack.iter().any(|d| d.name == self.name) {
return Ok(Some(tokens));
}
let mut stack_new: Vec<Definition> = stack.to_vec();
stack_new.push(self.clone());
if params.is_empty() {
tokens = Macro::resolve_all(&tokens, def_map, &stack_new, depth, mismatches, budget)?;
} else {
let mut local_map: HashMap<String, Definition> = HashMap::new();
for (key, value) in def_map {
local_map.insert(key.clone(), value.clone());
}
for (param, arg) in params.iter().zip(args.iter()) {
let mut tokens = parse_checked(
|deep| preprocess_grammar::tokens(arg, deep, false),
|e| error!("Failed to parse macro argument: {e}"),
)?;
let stack: Vec<Definition> = Vec::new();
tokens = Macro::resolve_all(&tokens, def_map, &stack, depth, mismatches, budget)?;
local_map.insert(
param.clone(),
Definition {
name: param.clone(),
parameters: None,
value: tokens,
local: true,
},
);
}
tokens =
Macro::resolve_all(&tokens, &local_map, &stack_new, depth, mismatches, budget)?;
}
Ok(Some(tokens))
}
}
impl Macro {
fn resolve_pseudoargs(
&self,
def_map: &HashMap<String, Definition>,
stack: &[Definition],
depth: usize,
mismatches: &mut Vec<String>,
budget: &mut Budget,
) -> Result<Vec<Token>, Error> {
let mut tokens: Vec<Token> = Vec::new();
tokens.push(Token::RegularToken(self.name.clone()));
if self.arguments.is_none() {
return Ok(tokens);
}
let (_, without_name) = self.original.split_at(self.name.len());
let mut arg_tokens = parse_checked(
|deep| preprocess_grammar::tokens(without_name, deep, false),
|e| error!("Failed to parse macro arguments: {e}"),
)?;
arg_tokens = Macro::resolve_all(&arg_tokens, def_map, stack, depth, mismatches, budget)?;
for t in arg_tokens {
tokens.push(t);
}
Ok(tokens)
}
fn resolve(
&self,
def_map: &HashMap<String, Definition>,
stack: &[Definition],
depth: usize,
mismatches: &mut Vec<String>,
budget: &mut Budget,
) -> Result<Vec<Token>, Error> {
if depth >= MAX_EXPANSION_DEPTH {
return Err(error!(
"Macro \"{}\" expands too deep (limit {MAX_EXPANSION_DEPTH})",
printable(&self.name)
));
}
let depth = depth + 1;
match def_map.get(&self.name) {
Some(def) => {
budget.step(&self.name)?;
if let Some(args) = &self.arguments {
if def.parameters.is_none() {
let mut tokens = def
.value(None, def_map, stack, depth, mismatches, budget)?
.unwrap_or_default();
let (_, rest) = self.original.split_at(self.name.len());
let mut rest_tokens = parse_checked(
|deep| preprocess_grammar::tokens(rest, deep, false),
|e| error!("Failed to parse macro arguments: {e}"),
)?;
rest_tokens = Macro::resolve_all(
&rest_tokens,
def_map,
stack,
depth,
mismatches,
budget,
)?;
tokens.extend(rest_tokens);
return Ok(tokens);
}
let expected = def.parameters.as_ref().map_or(0, std::vec::Vec::len);
let got = if expected == 0
&& args.len() == 1
&& args.first().is_some_and(|a| a.trim().is_empty())
{
0
} else {
args.len()
};
if got != expected {
let msg = format!(
"Macro \"{}\" expects {} argument(s), got {}",
printable(&self.name),
expected,
got
);
mismatches.push(msg);
return Ok(if self.quoted {
vec![Token::NewlineToken("\"\"".to_string(), 0)]
} else {
Vec::new()
});
}
}
let value = def.value(
self.arguments.as_deref(),
def_map,
stack,
depth,
mismatches,
budget,
)?;
if !def.local && self.quoted {
}
if let Some(tokens) = value {
if self.quoted {
budget.spend(
tokens.len(),
tokens.iter().map(token_bytes).sum(),
&self.name,
)?;
let (concatted, newlines) = Token::concat(&tokens);
let mut tokens: Vec<Token> = Vec::new();
tokens.push(Token::NewlineToken(
format!("\"{}\"", concatted.trim()),
newlines,
));
Ok(tokens)
} else {
Ok(tokens)
}
} else {
self.resolve_pseudoargs(def_map, stack, depth, mismatches, budget)
}
}
None => self.resolve_pseudoargs(def_map, stack, depth, mismatches, budget),
}
}
fn resolve_all(
tokens: &[Token],
def_map: &HashMap<String, Definition>,
stack: &[Definition],
depth: usize,
mismatches: &mut Vec<String>,
budget: &mut Budget,
) -> Result<Vec<Token>, Error> {
let mut result: Vec<Token> = Vec::new();
for token in tokens {
if let Token::MacroToken(m) = token {
let resolved = m.resolve(def_map, stack, depth, mismatches, budget)?;
budget.spend(
resolved.len(),
resolved.iter().map(token_bytes).sum(),
&m.name,
)?;
result.extend(resolved);
} else {
budget.spend(
1,
token_bytes(token),
stack.last().map_or("", |d| d.name.as_str()),
)?;
result.push(token.clone());
}
}
Ok(result)
}
}
impl Token {
fn concat(tokens: &[Token]) -> (String, u32) {
let mut output = String::new();
let mut newlines = 0;
for token in tokens {
match token {
Token::RegularToken(s) => {
output += s;
}
Token::NewlineToken(s, n) => {
output += s;
newlines += n;
}
Token::MacroToken(m) => {
output += &m.original;
}
Token::CommentToken(n) | Token::OpenCommentToken(n) => {
newlines += n;
}
Token::ConcatToken => {}
}
}
(output, newlines)
}
}
fn source_newlines(tokens: &[Token]) -> (u32, u32) {
let (mut inline, mut continuations) = (0u32, 0u32);
for token in tokens {
match token {
Token::NewlineToken(_, n) | Token::CommentToken(n) | Token::OpenCommentToken(n) => {
inline = inline.saturating_add(*n);
}
Token::MacroToken(m) => {
let mut in_string = false;
let mut prev = ' ';
for c in m.original.chars() {
match c {
'"' => in_string = !in_string,
'\n' if in_string => inline = inline.saturating_add(1),
'\n' if prev == '\\' => continuations = continuations.saturating_add(1),
_ => {}
}
prev = c;
}
}
Token::RegularToken(s) => {
let n = u32::try_from(s.matches("\\\n").count()).unwrap_or(u32::MAX);
continuations = continuations.saturating_add(n);
}
Token::ConcatToken => {}
}
}
(inline, continuations)
}
fn skipped_newlines(tokens: &[Token]) -> u32 {
let (inline, continuations) = source_newlines(tokens);
inline.saturating_add(continuations)
}
fn read_prefix(prefix_path: &Path) -> Result<String, Error> {
let mut content = String::new();
File::open(prefix_path)?.read_to_string(&mut content)?;
Ok(content.lines().next().unwrap_or_default().to_string())
}
#[must_use]
pub(crate) fn pathsep() -> &'static str {
if cfg!(windows) { "\\" } else { "/" }
}
fn names_equal(a: &OsStr, b: &OsStr, ci: bool) -> bool {
if ci {
a.to_string_lossy()
.eq_ignore_ascii_case(&b.to_string_lossy())
} else {
a == b
}
}
fn differs_in_case(a: &Path, b: &Path) -> bool {
let (mut a, mut b) = (a.components(), b.components());
loop {
match (a.next(), b.next()) {
(None, None) => return false,
(Some(x), Some(y)) if x.as_os_str() == y.as_os_str() => {}
_ => return true,
}
}
}
fn resolve_case_insensitive(base: &Path, relative: &Path) -> Option<PathBuf> {
let mut current = base.to_path_buf();
for component in relative.components() {
let Component::Normal(name) = component else {
continue;
};
let mut exact = None;
let mut ignoring_case = None;
for entry in read_dir(¤t).ok()?.filter_map(std::result::Result::ok) {
if exact.is_none() && names_equal(&entry.file_name(), name, false) {
exact = Some(entry);
} else if ignoring_case.is_none() && names_equal(&entry.file_name(), name, true) {
ignoring_case = Some(entry);
}
}
current = exact.or(ignoring_case)?.path();
}
Some(current)
}
fn matches_include_path(path: &Path, include_path: &str, ci: bool) -> bool {
let include_pathbuf = PathBuf::from(&include_path.replace('\\', pathsep()));
match (path.file_name(), include_pathbuf.file_name()) {
(Some(a), Some(b)) if names_equal(a, b, ci) => {}
_ => return false,
}
for parent in path.ancestors() {
if parent.is_file() {
continue;
}
let prefixpath = parent.join("$PBOPREFIX$");
if !prefixpath.is_file() {
continue;
}
let Ok(mut prefix) = read_prefix(&prefixpath) else {
continue;
};
prefix = if !prefix.is_empty() && !prefix.starts_with('\\') {
format!("\\{prefix}")
} else {
prefix
};
let prefix_pathbuf = PathBuf::from(prefix.replace('\\', pathsep()));
let Ok(relative) = path.strip_prefix(parent) else {
continue;
};
let test_path = prefix_pathbuf.join(relative);
let (mut a, mut b) = (test_path.components(), include_pathbuf.components());
loop {
match (a.next(), b.next()) {
(None, None) => return true,
(Some(x), Some(y)) if names_equal(x.as_os_str(), y.as_os_str(), ci) => {}
_ => break,
}
}
}
false
}
const MAX_SEARCH_ENTRIES: usize = 1_000_000;
struct Walk<'a> {
include_path: &'a str,
root: &'a Path,
ci: bool,
visited: usize,
limit: usize,
seen: std::collections::BTreeSet<PathBuf>,
}
impl Walk<'_> {
fn skips_link(&self, path: &Path) -> bool {
path.canonicalize()
.map_or(true, |target| !target.starts_with(self.root))
}
fn search(&mut self, directory: &Path) -> Result<Option<PathBuf>, Error> {
let Ok(canonical) = directory.canonicalize() else {
return Ok(None);
};
if !self.seen.insert(canonical) {
return Ok(None);
}
if let Ok(entries) = read_dir(directory) {
for entry in entries.filter_map(std::result::Result::ok) {
self.visited += 1;
if self.visited > self.limit {
return Err(error!(
"Include \"{}\" not found: the search below \"{}\" stopped after {} entries",
printable(self.include_path),
printable(&self.root.display().to_string()),
self.limit
));
}
let path = entry.path();
let is_link = entry.file_type().is_ok_and(|t| t.is_symlink());
if path.is_dir() {
if path.file_name() == Some(OsStr::new(".git")) {
continue;
}
if is_link && self.skips_link(&path) {
continue;
}
if let Some(path) = self.search(&path)? {
return Ok(Some(path));
}
} else if matches_include_path(&path, self.include_path, self.ci) {
return Ok(Some(path));
}
}
}
Ok(lookup_directly(self.include_path, directory, self.ci))
}
}
fn lookup_directly(include_path: &str, directory: &Path, ci: bool) -> Option<PathBuf> {
let relative = PathBuf::from(
include_path
.trim_start_matches('\\')
.replace('\\', pathsep()),
);
let direct_pathbuf = if ci {
resolve_case_insensitive(directory, &relative)?
} else {
let joined = directory.join(&relative);
if !joined.is_file() {
return None;
}
match resolve_case_insensitive(directory, &relative) {
Some(found) if differs_in_case(&joined, &found) => return None,
_ => return Some(joined),
}
};
if direct_pathbuf.is_file() {
return Some(direct_pathbuf);
}
None
}
fn canonicalize(path: &Path) -> PathBuf {
let mut result = PathBuf::new();
for component in path.components() {
match component {
Component::ParentDir => {
result.pop();
}
_ => {
result.push(component);
}
}
}
result
}
fn include_not_found(include_path: &str, origin: Option<&Path>, line: u32) -> Error {
match origin {
Some(origin_path) => error!(
"File \"{}\" included from \"{}:{}\" not found.",
printable(include_path),
printable(&origin_path.display().to_string()),
line
),
None => error!(
"Included file \"{}\" (line {}) not found.",
printable(include_path),
line
),
}
}
fn warn_include_case(include_path: &str, found: &Path, warnings: &mut dyn WarningSink) {
warnings.warn(Warning::new(
Some("include-case"),
format!(
"Include \"{}\" only matched \"{}\" ignoring case.",
printable(include_path),
printable(&found.display().to_string())
),
));
}
fn find_include_file(
include_path: &str,
origin: Option<&Path>,
folders: &IncludeFolders,
line: u32,
warnings: &mut dyn WarningSink,
) -> Result<PathBuf, Error> {
if include_path.starts_with('\\') {
let canonical = |search_path: &PathBuf| {
search_path.canonicalize().map_err(|e| {
error!(
"Include folder \"{}\" is not usable: {}",
search_path.display(),
e
)
})
};
let roots = folders
.search
.iter()
.map(canonical)
.collect::<Result<Vec<PathBuf>, Error>>()?;
let direct = folders
.direct
.iter()
.map(canonical)
.collect::<Result<Vec<PathBuf>, Error>>()?;
for ci in [false, true] {
let mut found = None;
for root in &roots {
let mut walk = Walk {
include_path,
root,
ci,
visited: 0,
limit: MAX_SEARCH_ENTRIES,
seen: std::collections::BTreeSet::new(),
};
found = walk.search(root)?;
if found.is_some() {
break;
}
}
if found.is_none() {
found = direct
.iter()
.find_map(|root| lookup_directly(include_path, root, ci));
}
if let Some(file_path) = found {
if ci {
warn_include_case(include_path, &file_path, warnings);
}
return Ok(file_path);
}
}
Err(include_not_found(include_path, origin, line))
} else {
let mut path = PathBuf::from(include_path.replace('\\', pathsep()));
if let Some(origin_path) = origin {
let absolute = PathBuf::from(origin_path).canonicalize()?;
let Some(origin_dir) = absolute.parent() else {
return Err(error!(
"Include origin \"{}\" has no parent directory.",
printable(&origin_path.display().to_string())
));
};
path = origin_dir.join(path);
} else {
path = current_dir()?.join(path);
}
let absolute = canonicalize(&path);
let root: PathBuf = absolute
.components()
.take_while(|c| !matches!(c, Component::Normal(_)))
.collect();
let relative: PathBuf = absolute
.components()
.skip_while(|c| !matches!(c, Component::Normal(_)))
.collect();
if absolute.is_file() {
if let Some(found) = resolve_case_insensitive(&root, &relative)
&& differs_in_case(&absolute, &found)
{
warn_include_case(include_path, &found, warnings);
}
return Ok(absolute);
}
match resolve_case_insensitive(&root, &relative) {
Some(found) if found.is_file() => {
warn_include_case(include_path, &found, warnings);
Ok(found)
}
_ => Err(include_not_found(include_path, origin, line)),
}
}
}
fn warn_unterminated(origin: Option<&Path>, line: u32, warnings: &mut dyn WarningSink) {
let mut warning = Warning::new(
Some("unterminated-comment"),
"A /* comment is never closed; the rest of the file is ignored.",
)
.at_line(line);
if let Some(origin) = origin {
warning = warning.in_file(origin.display().to_string());
}
warnings.warn(warning);
}
#[expect(
clippy::too_many_lines,
reason = "one branch per directive keeps the preprocessor loop readable"
)]
fn preprocess_rec(
input: &str,
origin: Option<&Path>,
definition_map: &mut HashMap<String, Definition>,
info: &mut PreprocessInfo,
includefolders: &IncludeFolders,
warnings: &mut dyn WarningSink,
) -> Result<String, Error> {
let input = input.replace("\r\n", "\n").replace('\r', "\n");
let deep = Cell::new(false);
let parsed = preprocess_grammar::file(&input, &deep, true);
if deep.get() {
return Err(nesting_error());
}
let lines = PreprocessParseErrorExt::format_error(parsed, origin, &input)?;
let mut output = String::new();
let mut original_lineno = 1;
let mut level = 0;
let mut level_true = 0;
let mut last_is_entry = false;
for line in lines {
last_is_entry = false;
'line: {
match line {
Line::DirectiveLine(dir, newlines, open) => {
if open {
warn_unterminated(origin, original_lineno, warnings);
}
original_lineno += newlines;
match dir {
Directive::IncludeDirective(path) => {
if level > level_true {
break 'line;
}
let memory = includefolders.memory.as_ref();
let (file_path, in_memory) = if let Some(m) = memory {
let (name, content) = m.find(&path, origin, original_lineno)?;
(PathBuf::from(name), Some(content))
} else {
(
find_include_file(
&path,
origin,
includefolders,
original_lineno,
warnings,
)?,
None,
)
};
let is_mem = memory.is_some();
let wanted = identity(&file_path, is_mem);
if info
.import_stack
.iter()
.any(|p| identity(p, is_mem) == wanted)
{
let chain: Vec<String> = info
.import_stack
.iter()
.chain(std::iter::once(&file_path))
.map(|p| printable(&p.display().to_string()))
.collect();
return Err(error!("Include cycle: {}", chain.join(" -> ")));
}
if info.include_depth >= MAX_INCLUDE_DEPTH {
return Err(error!(
"Include \"{}\" nests too deep (limit {MAX_INCLUDE_DEPTH})",
printable(&path)
));
}
info.includes += 1;
if info.includes > MAX_INCLUDES {
return Err(error!(
"Include \"{}\" is the {}th include; the limit is {MAX_INCLUDES} per preprocess call",
printable(&path),
info.includes
));
}
let content = if let Some(bytes) = in_memory {
info.include_bytes = info.include_bytes.saturating_add(bytes.len());
if info.include_bytes > MAX_INCLUDE_BYTES {
return Err(error!(
"Include \"{}\" brings the included files past {MAX_INCLUDE_BYTES} bytes",
printable(&path)
));
}
String::from_utf8(bytes.to_vec()).map_err(|e| {
error!(
"Include \"{}\" is not valid UTF-8: {e}",
printable(&path)
)
})?
} else {
let mut content = String::new();
File::open(&file_path)?.read_to_string(&mut content)?;
content
};
info.import_stack.push(file_path.clone());
info.include_depth += 1;
let result = preprocess_rec(
&content,
Some(&file_path),
definition_map,
info,
includefolders,
warnings,
)
.prepend_error(format!(
"Failed to preprocess include \"{}\":",
printable(&path)
))?;
info.import_stack.pop();
info.include_depth -= 1;
output += &result;
}
Directive::DefineDirective(def) => {
if let Some(at) = def
.value
.iter()
.position(|t| matches!(t, Token::OpenCommentToken(_)))
{
let before = def.value.get(..at).unwrap_or_default();
warn_unterminated(
origin,
original_lineno.saturating_add(skipped_newlines(before)),
warnings,
);
}
original_lineno += skipped_newlines(&def.value);
if level > level_true {
break 'line;
}
if definition_map.remove(&def.name).is_some() {
}
definition_map.insert(def.name.clone(), def);
}
Directive::UndefDirective(name) => {
if level > level_true {
break 'line;
}
definition_map.remove(&name);
}
Directive::IfDefDirective(name) => {
level_true += i32::from(
level_true == level && definition_map.contains_key(&name),
);
level += 1;
}
Directive::IfNDefDirective(name) => {
level_true += i32::from(
level_true == level && !definition_map.contains_key(&name),
);
level += 1;
}
Directive::ElseDirective => {
if level_true + 1 == level {
level_true = level;
} else if level_true == level {
level_true -= 1;
}
}
Directive::EndIfDirective => {
if level == 0 {
return Err(error!("#endif without matching #ifdef/#ifndef"));
}
level -= 1;
if level_true > level {
level_true -= 1;
}
}
}
}
Line::TokenLine(tokens) => {
if let Some(at) = tokens
.iter()
.position(|t| matches!(t, Token::OpenCommentToken(_)))
{
let before = tokens.get(..at).unwrap_or_default();
warn_unterminated(
origin,
original_lineno.saturating_add(skipped_newlines(before)),
warnings,
);
}
if level > level_true {
original_lineno += skipped_newlines(&tokens);
break 'line;
}
last_is_entry = !tokens
.iter()
.any(|t| matches!(t, Token::OpenCommentToken(_)))
&& tokens.iter().any(|t| match t {
Token::RegularToken(s) => !s.trim().is_empty(),
Token::NewlineToken(..) | Token::MacroToken(_) => true,
_ => false,
});
let stack: Vec<Definition> = Vec::new();
let mut mismatches: Vec<String> = Vec::new();
let mut budget = Budget::new(info.steps);
let resolved = Macro::resolve_all(
&tokens,
definition_map,
&stack,
0,
&mut mismatches,
&mut budget,
)
.prepend_error("Failed to resolve macros:");
let resolved = resolved?;
info.steps = info.steps.saturating_add(budget.steps);
info.expanded = info.expanded.saturating_add(budget.tokens);
info.expanded_bytes = info.expanded_bytes.saturating_add(budget.bytes);
if info.expanded > MAX_FILE_EXPANSION || info.expanded_bytes > MAX_FILE_BYTES {
return Err(error!(
"Macro expansion produced more than {MAX_FILE_EXPANSION} tokens or {MAX_FILE_BYTES} bytes in one file and its includes"
));
}
let (inline, continuations) = source_newlines(&tokens);
let (mut result, _) = Token::concat(&resolved);
result = result.replace("\r\n", "\n");
original_lineno += inline;
result = result.replace("\\\n", "");
let file = origin.map(|p| p.display().to_string());
for msg in mismatches {
let mut warning =
Warning::new(Some("macro-arg-count"), msg).at_line(original_lineno);
if let Some(file) = &file {
warning = warning.in_file(file.clone());
}
warnings.warn(warning);
}
output += &result;
output += "\n";
info.line_origins
.push((original_lineno, origin.map(Path::to_path_buf)));
original_lineno += continuations;
}
}
}
original_lineno += 1;
if level > 0 {
}
}
info.ends_in_entry = last_is_entry;
Ok(output)
}
pub fn preprocess(
input: String,
origin: Option<PathBuf>,
includefolders: &[PathBuf],
warnings: &mut dyn WarningSink,
) -> Result<(String, PreprocessInfo), Error> {
preprocess_in(
input,
origin,
&IncludeFolders::new(includefolders.to_vec()),
warnings,
)
}
#[expect(
clippy::needless_pass_by_value,
reason = "public API takes ownership of the input; callers pass owned strings"
)]
pub fn preprocess_in(
mut input: String,
origin: Option<PathBuf>,
includefolders: &IncludeFolders,
warnings: &mut dyn WarningSink,
) -> Result<(String, PreprocessInfo), Error> {
if input.as_bytes().starts_with(&[0xef, 0xbb, 0xbf]) {
input = input[3..].to_string();
}
for folder in includefolders
.search
.iter()
.chain(&includefolders.direct)
.filter(|_| includefolders.memory.is_none())
{
if !folder.is_dir() {
return Err(error!(
"Include folder \"{}\" does not exist or is not a directory.",
folder.display()
));
}
}
let mut info = PreprocessInfo {
line_origins: Vec::new(),
import_stack: Vec::new(),
include_depth: 0,
expanded: 0,
expanded_bytes: 0,
steps: 0,
includes: 0,
include_bytes: 0,
ends_in_entry: false,
};
if let Some(ref path) = origin {
info.import_stack.push(path.clone());
}
let mut def_map: HashMap<String, Definition> = HashMap::new();
match preprocess_rec(
&input,
origin.as_deref(),
&mut def_map,
&mut info,
includefolders,
warnings,
) {
Ok(result) => Ok((result, info)),
Err(e) => Err(e),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(unix)]
#[test]
fn sibling_links_are_not_visited_twice() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().canonicalize().unwrap();
std::fs::create_dir(root.join("real")).unwrap();
std::fs::write(root.join("real/f.txt"), "x").unwrap();
std::os::unix::fs::symlink("real", root.join("l1")).unwrap();
std::os::unix::fs::symlink("real", root.join("l2")).unwrap();
let mut walk = Walk {
include_path: "\\DZ\\gone.hpp",
root: &root,
ci: false,
visited: 0,
limit: 1000,
seen: std::collections::BTreeSet::new(),
};
assert!(walk.search(&root).unwrap().is_none());
assert_eq!(walk.visited, 4);
}
#[test]
fn search_stops_after_the_entry_limit() {
let dir = tempfile::tempdir().unwrap();
for i in 0..10 {
std::fs::write(dir.path().join(format!("f{i}.txt")), "x").unwrap();
}
let root = dir.path().canonicalize().unwrap();
let mut walk = Walk {
include_path: "\\DZ\\gone.hpp",
root: &root,
ci: false,
visited: 0,
limit: 5,
seen: std::collections::BTreeSet::new(),
};
let err = walk.search(&root).unwrap_err().to_string();
assert!(
err.contains("DZ") && err.contains("stopped after 5 entries"),
"{err}"
);
}
use std::ffi::OsStr;
#[test]
fn arg_count_mismatches_belong_to_their_call() {
let mut defs: HashMap<String, Definition> = HashMap::new();
let def = Definition {
name: "F".to_string(),
parameters: Some(vec!["a".to_string()]),
value: Vec::new(),
local: false,
};
defs.insert("F".to_string(), def);
let tokens = vec![Token::MacroToken(parse_macro("F(1,2)").unwrap())];
let mut first = Vec::new();
Macro::resolve_all(&tokens, &defs, &[], 0, &mut first, &mut Budget::new(0)).unwrap();
assert_eq!(first.len(), 1);
let plain = vec![Token::RegularToken("x".to_string())];
let mut second = Vec::new();
Macro::resolve_all(&plain, &defs, &[], 0, &mut second, &mut Budget::new(0)).unwrap();
assert_eq!(second.len(), 0);
}
#[test]
fn resolve_reports_the_on_disk_casing() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("inc.hpp"), b"x").unwrap();
let found = resolve_case_insensitive(dir.path(), Path::new("INC.hpp")).unwrap();
assert_eq!(found.file_name(), Some(OsStr::new("inc.hpp")));
}
#[test]
#[cfg(not(windows))]
fn resolve_prefers_the_exact_match() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("Inc.hpp"), b"a").unwrap();
std::fs::write(dir.path().join("inc.hpp"), b"b").unwrap();
let found = resolve_case_insensitive(dir.path(), Path::new("inc.hpp")).unwrap();
assert_eq!(found.file_name(), Some(OsStr::new("inc.hpp")));
}
#[test]
fn case_difference_is_detected_component_wise() {
assert!(differs_in_case(
Path::new("/a/B.hpp"),
Path::new("/a/b.hpp")
));
assert!(!differs_in_case(
Path::new("/a/b.hpp"),
Path::new("/a/b.hpp")
));
}
}