#![allow(
clippy::enum_glob_use,
clippy::if_not_else,
clippy::too_many_lines,
clippy::wildcard_imports
)]
use std::collections::{HashMap, HashSet, hash_map};
use std::path::{Path, PathBuf};
use petgraph::{
Direction, algo::kosaraju_scc, graph::NodeIndex, stable_graph::StableGraph, visit::Dfs,
};
use serde::{Deserialize, Serialize};
use crate::c_langs_macros::is_specials;
use crate::langs::*;
use crate::languages::language_preproc::*;
use crate::node::{Cursor, Node};
use crate::tools::*;
use crate::traits::*;
#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
pub enum PreprocDiagnostic {
SelfInclusion {
file: PathBuf,
},
IncludeCycle {
members: Vec<String>,
},
NonUtf8CyclePath {
path: String,
},
NonUtf8IndirectInclude {
path: String,
},
NotPreprocessed {
file: PathBuf,
},
}
impl std::fmt::Display for PreprocDiagnostic {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::SelfInclusion { file } => {
write!(f, "Warning: possible self inclusion {}", file.display())
}
Self::IncludeCycle { members } => {
writeln!(f, "Warning: possible include cycle:")?;
for member in members {
writeln!(f, " - \"{member}\"")?;
}
Ok(())
}
Self::NonUtf8CyclePath { path } => {
write!(
f,
"warning: skipping non-UTF-8 path in include cycle: {path}"
)
}
Self::NonUtf8IndirectInclude { path } => write!(
f,
"warning: skipping non-UTF-8 indirect include path: {path}"
),
Self::NotPreprocessed { file } => write!(
f,
"Warning: included file which has not been preprocessed: {}",
file.display()
),
}
}
}
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct PreprocFile {
pub direct_includes: HashSet<String>,
pub indirect_includes: HashSet<String>,
pub macros: HashSet<String>,
}
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct PreprocResults {
pub files: HashMap<PathBuf, PreprocFile>,
}
impl PreprocFile {
#[must_use]
pub fn new_macros(macros: &[&str]) -> Self {
let mut pf = Self::default();
for m in macros {
pf.macros.insert((*m).to_string());
}
pf
}
}
pub fn get_macros<S: ::std::hash::BuildHasher>(
file: &Path,
files: &HashMap<PathBuf, PreprocFile, S>,
) -> HashSet<String> {
let mut macros = HashSet::new();
if let Some(pf) = files.get(file) {
for m in &pf.macros {
macros.insert(m.clone());
}
for f in &pf.indirect_includes {
if let Some(pf) = files.get(&PathBuf::from(f)) {
for m in &pf.macros {
macros.insert(m.clone());
}
}
}
}
macros
}
type IncludeGraph = StableGraph<PathBuf, i32>;
fn ensure_node(
g: &mut IncludeGraph,
nodes: &mut HashMap<PathBuf, NodeIndex>,
file: &Path,
) -> NodeIndex {
match nodes.entry(file.to_path_buf()) {
hash_map::Entry::Occupied(l) => *l.get(),
hash_map::Entry::Vacant(p) => *p.insert(g.add_node(file.to_path_buf())),
}
}
fn resolve_single_include<S: ::std::hash::BuildHasher>(
file: &Path,
include: &str,
all_files: &HashMap<String, Vec<PathBuf>, S>,
) -> Option<PathBuf> {
guess_file(file, include, all_files).into_iter().min()
}
fn build_include_graph<S: ::std::hash::BuildHasher>(
files: &HashMap<PathBuf, PreprocFile, S>,
all_files: &HashMap<String, Vec<PathBuf>, S>,
diagnostics: &mut Vec<PreprocDiagnostic>,
) -> (IncludeGraph, HashMap<PathBuf, NodeIndex>) {
let mut nodes: HashMap<PathBuf, NodeIndex> = HashMap::new();
let mut g = StableGraph::new();
for (file, pf) in files {
let node = ensure_node(&mut g, &mut nodes, file);
for i in &pf.direct_includes {
let Some(included) = resolve_single_include(file, i, all_files) else {
continue;
};
if &included == file {
diagnostics.push(PreprocDiagnostic::SelfInclusion { file: file.clone() });
continue;
}
let included = ensure_node(&mut g, &mut nodes, &included);
g.add_edge(node, included, 0);
}
}
(g, nodes)
}
fn scc_external_neighbors(
g: &IncludeGraph,
component: &[NodeIndex],
direction: Direction,
) -> Vec<NodeIndex> {
let mut neighbors = Vec::new();
for c in component {
for n in g.neighbors_directed(*c, direction) {
if !component.contains(&n) && !neighbors.contains(&n) {
neighbors.push(n);
}
}
}
neighbors
}
fn collapse_scc(
g: &mut IncludeGraph,
nodes: &mut HashMap<PathBuf, NodeIndex>,
diagnostics: &mut Vec<PreprocDiagnostic>,
) -> HashMap<NodeIndex, HashSet<String>> {
let mut scc = kosaraju_scc(&*g);
let mut scc_map: HashMap<NodeIndex, HashSet<String>> = HashMap::new();
for component in &mut scc {
if component.len() > 1 {
let incoming = scc_external_neighbors(g, component, Direction::Incoming);
let outgoing = scc_external_neighbors(g, component, Direction::Outgoing);
let mut paths = HashSet::new();
let replacement = g.add_node(PathBuf::from(""));
for i in incoming {
g.add_edge(i, replacement, 0);
}
for o in outgoing {
g.add_edge(replacement, o, 0);
}
for c in component.drain(..) {
let path = g
.remove_node(c)
.expect("invariant: SCC component node must exist in graph");
if let Some(s) = path.to_str() {
paths.insert(s.to_string());
} else {
diagnostics.push(PreprocDiagnostic::NonUtf8CyclePath {
path: path.display().to_string(),
});
}
*nodes
.get_mut(&path)
.expect("invariant: every graph node must have a nodes map entry") =
replacement;
}
let mut members: Vec<String> = paths.iter().cloned().collect();
members.sort_unstable();
diagnostics.push(PreprocDiagnostic::IncludeCycle { members });
scc_map.insert(replacement, paths);
}
}
scc_map
}
fn record_indirect_includes<S: ::std::hash::BuildHasher>(
files: &mut HashMap<PathBuf, PreprocFile, S>,
g: &IncludeGraph,
nodes: &HashMap<PathBuf, NodeIndex>,
scc_map: &HashMap<NodeIndex, HashSet<String>>,
diagnostics: &mut Vec<PreprocDiagnostic>,
) {
for (path, start) in nodes {
let Some(pf) = files.get_mut(path) else {
diagnostics.push(PreprocDiagnostic::NotPreprocessed { file: path.clone() });
continue;
};
accumulate_reachable_includes(g, *start, scc_map, &mut pf.indirect_includes, diagnostics);
}
}
fn accumulate_reachable_includes(
g: &IncludeGraph,
start: NodeIndex,
scc_map: &HashMap<NodeIndex, HashSet<String>>,
x_inc: &mut HashSet<String>,
diagnostics: &mut Vec<PreprocDiagnostic>,
) {
let mut dfs = Dfs::new(g, start);
while let Some(node) = dfs.next(g) {
let w = g
.node_weight(node)
.expect("invariant: DFS-visited node must have weight in graph");
if w == &PathBuf::from("") {
let paths = scc_map.get(&node).expect(
"every empty-path node is an SCC replacement and must have a scc_map entry",
);
x_inc.extend(paths.iter().cloned());
} else if let Some(s) = w.to_str() {
x_inc.insert(s.to_string());
} else {
diagnostics.push(PreprocDiagnostic::NonUtf8IndirectInclude {
path: w.display().to_string(),
});
}
}
}
pub fn fix_includes<S: ::std::hash::BuildHasher>(
files: &mut HashMap<PathBuf, PreprocFile, S>,
all_files: &HashMap<String, Vec<PathBuf>, S>,
) -> Vec<PreprocDiagnostic> {
let mut diagnostics = Vec::new();
let (mut g, mut nodes) = build_include_graph(files, all_files, &mut diagnostics);
let scc_map = collapse_scc(&mut g, &mut nodes, &mut diagnostics);
record_indirect_includes(files, &g, &nodes, &scc_map, &mut diagnostics);
diagnostics
}
fn strip_include_quotes(code: &[u8], start: usize, end: usize) -> Option<&str> {
const MIN_QUOTED_LEN: usize = 2;
if end < start + MIN_QUOTED_LEN {
return None;
}
let inner = &code[start + 1..end - 1];
let first = inner.iter().position(|&c| c != b' ' && c != b'\t')?;
let last = inner.iter().rposition(|&c| c != b' ' && c != b'\t')?;
std::str::from_utf8(&inner[first..=last]).ok()
}
pub fn preprocess(source: Vec<u8>, path: &Path, results: &mut PreprocResults) {
preprocess_with_parser(&PreprocParser::new(source, path, None), path, results);
}
pub(crate) fn preprocess_with_parser(
parser: &PreprocParser,
path: &Path,
results: &mut PreprocResults,
) {
let node = parser.root();
let mut cursor = node.cursor();
let code = parser.code();
let mut file_result = PreprocFile::default();
let mut macro_events: Vec<(usize, MacroEvent)> = Vec::new();
let mut stack = vec![node];
while let Some(node) = stack.pop() {
push_children(&mut cursor, &node, &mut stack);
classify_preproc_node(
&mut cursor,
&node,
code,
&mut file_result,
&mut macro_events,
);
}
apply_macro_events(macro_events, &mut file_result);
results.files.insert(path.to_path_buf(), file_result);
}
fn push_children<'a>(cursor: &mut Cursor<'a>, node: &Node<'a>, stack: &mut Vec<Node<'a>>) {
cursor.reset(node);
if cursor.goto_first_child() {
loop {
stack.push(cursor.node());
if !cursor.goto_next_sibling() {
break;
}
}
}
}
fn classify_preproc_node<'a>(
cursor: &mut Cursor<'a>,
node: &Node<'a>,
code: &'a [u8],
file_result: &mut PreprocFile,
macro_events: &mut Vec<(usize, MacroEvent)>,
) {
let id = Preproc::from(node.kind_id());
match id {
Preproc::Define | Preproc::Undef => {
cursor.reset(node);
cursor.goto_first_child();
let identifier = cursor.node();
if identifier.kind_id() == Preproc::Identifier
&& let Some(macro_text) = identifier.utf8_text(code)
&& !is_specials(macro_text)
{
let event = if id == Preproc::Undef {
MacroEvent::Undef(macro_text.to_string())
} else {
MacroEvent::Define(macro_text.to_string())
};
macro_events.push((identifier.start_byte(), event));
}
}
Preproc::PreprocInclude => {
cursor.reset(node);
cursor.goto_first_child();
let file = cursor.node();
if file.kind_id() == Preproc::StringLiteral
&& let Some(include) =
strip_include_quotes(code, file.start_byte(), file.end_byte())
{
file_result.direct_includes.insert(include.to_string());
}
}
_ => {}
}
}
fn apply_macro_events(mut macro_events: Vec<(usize, MacroEvent)>, file_result: &mut PreprocFile) {
macro_events.sort_by_key(|(offset, _)| *offset);
for (_, event) in macro_events {
match event {
MacroEvent::Define(name) => {
file_result.macros.insert(name);
}
MacroEvent::Undef(name) => {
file_result.macros.remove(&name);
}
}
}
}
enum MacroEvent {
Define(String),
Undef(String),
}
#[cfg(test)]
#[allow(
clippy::float_cmp,
clippy::cast_precision_loss,
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
clippy::similar_names,
clippy::doc_markdown,
clippy::needless_raw_string_hashes,
clippy::too_many_lines
)]
mod tests {
use super::*;
fn parse(source: &str) -> PreprocParser {
PreprocParser::new(source.as_bytes().to_vec(), &PathBuf::from("test.h"), None)
}
#[test]
fn preprocess_empty_include_does_not_panic() {
let parser = parse("#include \"\"\n");
let mut results = PreprocResults::default();
preprocess_with_parser(&parser, &PathBuf::from("test.h"), &mut results);
let pf = results
.files
.get(&PathBuf::from("test.h"))
.expect("file entry must be inserted");
assert!(pf.direct_includes.is_empty());
}
#[test]
fn preprocess_whitespace_only_include_does_not_panic() {
let parser = parse("#include \" \"\n");
let mut results = PreprocResults::default();
preprocess_with_parser(&parser, &PathBuf::from("test.h"), &mut results);
let pf = results
.files
.get(&PathBuf::from("test.h"))
.expect("file entry must be inserted");
assert!(pf.direct_includes.is_empty());
}
#[test]
fn preprocess_valid_include_is_recorded() {
let parser = parse("#include \" foo.h \"\n");
let mut results = PreprocResults::default();
preprocess_with_parser(&parser, &PathBuf::from("test.h"), &mut results);
let pf = results
.files
.get(&PathBuf::from("test.h"))
.expect("file entry must be inserted");
assert!(pf.direct_includes.contains("foo.h"));
}
#[test]
fn preprocess_define_records_macro() {
let parser = parse("#define FOO 1\n");
let mut results = PreprocResults::default();
preprocess_with_parser(&parser, &PathBuf::from("test.h"), &mut results);
let pf = results
.files
.get(&PathBuf::from("test.h"))
.expect("file entry must be inserted");
assert!(pf.macros.contains("FOO"));
}
fn macros_of(source: &str) -> HashSet<String> {
let parser = parse(source);
let mut results = PreprocResults::default();
preprocess_with_parser(&parser, &PathBuf::from("test.h"), &mut results);
results
.files
.get(&PathBuf::from("test.h"))
.expect("file entry must be inserted")
.macros
.clone()
}
#[test]
fn preprocess_undef_removes_defined_macro() {
let macros = macros_of("#define FOO 1\n#undef FOO\n");
assert!(
!macros.contains("FOO"),
"#undef FOO must un-define FOO; got {macros:?}"
);
}
#[test]
fn preprocess_undef_of_never_defined_is_noop() {
let macros = macros_of("#undef NEVER_DEFINED\n");
assert!(!macros.contains("NEVER_DEFINED"));
}
#[test]
fn preprocess_define_after_undef_reintroduces_in_source_order() {
let macros = macros_of("#undef FOO\n#define FOO 1\n");
assert!(
macros.contains("FOO"),
"the trailing source-order #define must win; got {macros:?}"
);
}
#[test]
fn preprocess_undef_leaves_other_macros() {
let macros = macros_of("#define FOO 1\n#define BAR 2\n#undef FOO\n");
assert!(!macros.contains("FOO"));
assert!(macros.contains("BAR"));
}
#[test]
fn preprocess_define_of_special_token_is_skipped() {
let macros = macros_of("#define size_t unsigned\n#define APP_FLAG 1\n");
assert!(
!macros.contains("size_t"),
"special token `size_t` must be filtered out; got {macros:?}"
);
assert!(
macros.contains("APP_FLAG"),
"an ordinary adjacent macro must still be recorded; got {macros:?}"
);
}
#[test]
fn ambiguous_include_resolves_to_single_deterministic_candidate() {
let includer = PathBuf::from("proj/src/main.c");
let cfg_a = PathBuf::from("proj/aaa/config.h");
let cfg_b = PathBuf::from("proj/zzz/config.h");
let mut files: HashMap<PathBuf, PreprocFile> = HashMap::new();
let mut main = PreprocFile::default();
main.direct_includes.insert("config.h".to_string());
files.insert(includer.clone(), main);
files.insert(cfg_a.clone(), PreprocFile::new_macros(&["FROM_A"]));
files.insert(cfg_b.clone(), PreprocFile::new_macros(&["FROM_B"]));
let mut all_files: HashMap<String, Vec<PathBuf>> = HashMap::new();
all_files.insert("config.h".to_string(), vec![cfg_b.clone(), cfg_a.clone()]);
all_files.insert("main.c".to_string(), vec![includer.clone()]);
let diagnostics = fix_includes(&mut files, &all_files);
assert!(
diagnostics.is_empty(),
"no diagnostics expected for a clean ambiguous resolve; got {diagnostics:?}"
);
let main = files.get(&includer).expect("main.c retained");
assert!(main.indirect_includes.contains("proj/aaa/config.h"));
assert!(!main.indirect_includes.contains("proj/zzz/config.h"));
let macros = get_macros(&includer, &files);
assert!(macros.contains("FROM_A"));
assert!(
!macros.contains("FROM_B"),
"macros from the unselected candidate must not leak; got {macros:?}"
);
}
#[test]
fn self_inclusion_is_reported_as_diagnostic() {
let self_path = PathBuf::from("a.h");
let mut files: HashMap<PathBuf, PreprocFile> = HashMap::new();
let mut a = PreprocFile::default();
a.direct_includes.insert("a.h".to_string());
files.insert(self_path.clone(), a);
let mut all_files: HashMap<String, Vec<PathBuf>> = HashMap::new();
all_files.insert("a.h".to_string(), vec![self_path.clone()]);
let diagnostics = fix_includes(&mut files, &all_files);
assert_eq!(
diagnostics,
vec![PreprocDiagnostic::SelfInclusion {
file: self_path.clone(),
}]
);
}
#[test]
fn fix_includes_handles_simple_cycle() {
let mut files: HashMap<PathBuf, PreprocFile> = HashMap::new();
let mut a = PreprocFile::default();
a.direct_includes.insert("b.h".to_string());
let mut b = PreprocFile::default();
b.direct_includes.insert("a.h".to_string());
files.insert(PathBuf::from("a.h"), a);
files.insert(PathBuf::from("b.h"), b);
let mut all_files: HashMap<String, Vec<PathBuf>> = HashMap::new();
all_files.insert("a.h".to_string(), vec![PathBuf::from("a.h")]);
all_files.insert("b.h".to_string(), vec![PathBuf::from("b.h")]);
let diagnostics = fix_includes(&mut files, &all_files);
assert_eq!(
diagnostics,
vec![PreprocDiagnostic::IncludeCycle {
members: vec!["a.h".to_string(), "b.h".to_string()],
}]
);
let a = files
.get(&PathBuf::from("a.h"))
.expect("a.h must be retained");
assert!(a.indirect_includes.contains("a.h"));
assert!(a.indirect_includes.contains("b.h"));
let b = files
.get(&PathBuf::from("b.h"))
.expect("b.h must be retained");
assert!(b.indirect_includes.contains("a.h"));
assert!(b.indirect_includes.contains("b.h"));
}
#[test]
fn ensure_node_returns_stable_index_on_repeat() {
let mut g: IncludeGraph = StableGraph::new();
let mut nodes: HashMap<PathBuf, NodeIndex> = HashMap::new();
let p = PathBuf::from("a.h");
let first = ensure_node(&mut g, &mut nodes, &p);
let second = ensure_node(&mut g, &mut nodes, &p);
assert_eq!(first, second);
assert_eq!(g.node_count(), 1);
assert_eq!(nodes.len(), 1);
}
#[test]
fn scc_external_neighbors_dedups_and_excludes_intra_component() {
let mut graph: IncludeGraph = StableGraph::new();
let member_a = graph.add_node(PathBuf::from("a.h"));
let member_b = graph.add_node(PathBuf::from("b.h"));
let pred = graph.add_node(PathBuf::from("x.h"));
let succ = graph.add_node(PathBuf::from("y.h"));
graph.add_edge(member_a, member_b, 0);
graph.add_edge(member_b, member_a, 0);
graph.add_edge(pred, member_a, 0);
graph.add_edge(pred, member_b, 0);
graph.add_edge(member_a, succ, 0);
graph.add_edge(member_b, succ, 0);
let component = vec![member_a, member_b];
let incoming = scc_external_neighbors(&graph, &component, Direction::Incoming);
let outgoing = scc_external_neighbors(&graph, &component, Direction::Outgoing);
assert_eq!(incoming, vec![pred]);
assert_eq!(outgoing, vec![succ]);
}
#[test]
fn strip_include_quotes_rejects_too_short_spans() {
let code = b"#include \"\"";
assert_eq!(strip_include_quotes(code, 9, 9), None);
assert_eq!(strip_include_quotes(code, 9, 10), None);
}
#[test]
fn strip_include_quotes_handles_valid_and_empty_payloads() {
let code = b"#include \" foo.h \"";
assert_eq!(strip_include_quotes(code, 9, code.len()), Some("foo.h"));
let code = b"#include \"\"";
assert_eq!(strip_include_quotes(code, 9, 11), None);
let code = b"#include \" \"";
assert_eq!(strip_include_quotes(code, 9, 14), None);
}
#[test]
fn preprocess_truncated_include_does_not_panic() {
let parser = parse("#include \"\n");
let mut results = PreprocResults::default();
preprocess_with_parser(&parser, &PathBuf::from("test.h"), &mut results);
let pf = results
.files
.get(&PathBuf::from("test.h"))
.expect("file entry must be inserted");
assert!(pf.direct_includes.is_empty());
}
}