use std::collections::HashSet;
use std::path::{Path, PathBuf};
use cpp_parser::Dialect;
use crate::cache::cache_db::CacheKey;
use crate::cache::{SummaryKey, content_hash, fnv1a64};
use crate::file::paths::{DiskFiles, FileProvider, normalize_path};
use crate::include::config::CompilerConfig;
use crate::index::project::ProjectIndex;
use crate::index::{CookedFile, FileIndexer};
use crate::summary::{FileSummary, IncludeFact};
use crate::{IncludeForm, cache};
pub struct SummaryStore<F: FileProvider = DiskFiles> {
root: PathBuf,
cache: PathBuf,
config: CompilerConfig,
files: F,
index: ProjectIndex,
stats: StoreStats,
revision: std::sync::atomic::AtomicU64,
dead_macros: std::sync::Mutex<std::collections::HashMap<String, Vec<usize>>>,
refined_marks: std::sync::Mutex<std::collections::HashSet<String>>,
summary_database: std::sync::OnceLock<Option<std::sync::Arc<cache::cache_db::SummaryCache>>>,
generation: u64,
include_searches: std::sync::Mutex<
std::collections::HashMap<(IncludeForm, bool, Box<str>, PathBuf), Option<PathBuf>>,
>,
walk_cache: crate::index::walk_cache::WalkCache,
directives: std::sync::Mutex<
std::collections::HashMap<String, (u64, std::sync::Arc<crate::FileSummary>)>,
>,
}
const MAX_DIRECTIVE_READINGS: usize = 512;
const MAX_INCLUDE_SEARCHES: usize = 8192;
#[derive(Debug, Default)]
pub struct ServedRefinements {
pub still_to_do: Vec<std::path::PathBuf>,
pub served: Vec<std::path::PathBuf>,
}
pub struct ReRead {
pub rebuilt: Vec<(FileSummary, bool)>,
pub cooked: Vec<(PathBuf, crate::CookedFile)>,
not_yet: Vec<PathBuf>,
}
impl ReRead {
pub fn is_empty(&self) -> bool {
self.rebuilt.is_empty()
}
pub fn waiting_for_the_index(&self) -> &[PathBuf] {
&self.not_yet
}
}
const PARALLEL_WORKERS: usize = 8;
static PREPARED: [std::sync::atomic::AtomicU64; 5] = [
std::sync::atomic::AtomicU64::new(0),
std::sync::atomic::AtomicU64::new(0),
std::sync::atomic::AtomicU64::new(0),
std::sync::atomic::AtomicU64::new(0),
std::sync::atomic::AtomicU64::new(0),
];
fn note_a_prepared_file(
read: std::time::Duration,
decode: std::time::Duration,
verify: std::time::Duration,
build: std::time::Duration,
) {
use std::sync::atomic::Ordering::Relaxed;
let nanos =
|duration: std::time::Duration| duration.as_nanos().min(u128::from(u64::MAX)) as u64;
PREPARED[0].fetch_add(1, Relaxed);
PREPARED[1].fetch_add(nanos(read), Relaxed);
PREPARED[2].fetch_add(nanos(decode), Relaxed);
PREPARED[3].fetch_add(nanos(verify), Relaxed);
PREPARED[4].fetch_add(nanos(build), Relaxed);
}
fn prepared_stages() -> [u64; 5] {
use std::sync::atomic::Ordering::Relaxed;
[
PREPARED[0].load(Relaxed),
PREPARED[1].load(Relaxed),
PREPARED[2].load(Relaxed),
PREPARED[3].load(Relaxed),
PREPARED[4].load(Relaxed),
]
}
const WORKER_STACK: usize = 16 * 1024 * 1024;
fn parallel_map<T, R>(items: &[T], work: impl Fn(&T) -> R + Sync) -> Vec<R>
where
T: Sync,
R: Send,
{
let workers = std::thread::available_parallelism()
.map_or(1, |cores| cores.get())
.min(PARALLEL_WORKERS)
.min(items.len());
if workers <= 1 {
return items.iter().map(work).collect();
}
let next = std::sync::atomic::AtomicUsize::new(0);
let slots: Vec<std::sync::Mutex<Option<R>>> =
items.iter().map(|_| std::sync::Mutex::new(None)).collect();
std::thread::scope(|scope| {
for _ in 0..workers {
let worker = || {
loop {
let at = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let Some(item) = items.get(at) else {
break;
};
let made = work(item);
*slots[at]
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(made);
}
};
let _ = std::thread::Builder::new()
.name("cppls-index".to_string())
.stack_size(WORKER_STACK)
.spawn_scoped(scope, worker);
}
});
slots
.into_iter()
.zip(items)
.map(|(slot, item)| {
slot.into_inner()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.unwrap_or_else(|| work(item))
})
.collect()
}
pub struct Prepared {
outcome: PreparedOutcome,
with_environment: bool,
rendering: Option<std::sync::Arc<crate::preprocess::cooked::RenderedCooked>>,
}
impl Prepared {
pub(crate) fn heap_bytes(&self) -> usize {
let summary = match &self.outcome {
PreparedOutcome::Built { summary, .. } => summary.heap_bytes(),
_ => 0,
};
summary
+ self
.rendering
.as_ref()
.map_or(0, |rendered| rendered.weight())
}
pub(crate) fn with_environment(&self) -> bool {
self.with_environment
}
pub(crate) fn rendering(
&self,
) -> Option<(
u64,
std::sync::Arc<crate::preprocess::cooked::RenderedCooked>,
)> {
let PreparedOutcome::Built { summary, .. } = &self.outcome else {
return None;
};
Some((summary.key.content_hash, self.rendering.as_ref()?.clone()))
}
}
pub(crate) struct DetachedRead<F: FileProvider> {
path: PathBuf,
source: String,
config: CompilerConfig,
files: F,
seed: crate::Marked,
key: SummaryKey,
dead_macros: Vec<usize>,
timeline: Option<std::sync::Arc<crate::TranslationUnit>>,
}
impl<F: FileProvider> DetachedRead<F> {
pub(crate) fn read(self) -> Prepared {
let indexer = FileIndexer::new(&self.files, &self.config)
.with_seed(&self.seed)
.without_these_macros(&self.dead_macros);
let (summary, cooked, rendering) = match &self.timeline {
Some(unit) => {
let view = unit
.environment_of(&self.path)
.expect("the captured timeline contains the file");
let definitions = unit.definitions();
let seed = crate::macros::MacroTable::from_marked(&self.seed);
let (tokens, _) =
cpp_parser::lex(&self.source, &cpp_parser::LexerConfig::default());
let macros = crate::preprocess::cooked::FileMacros::new(
&view,
&definitions,
Some(&seed),
true,
);
let rendered =
crate::preprocess::cooked::cook_with(&self.source, &tokens, ¯os).render();
let indexed =
indexer.index_the_readings(&self.path, &self.source, &rendered, self.key);
let summary = indexed.summary.clone();
let rendering = indexed.rendered.clone();
(summary, Some(CookedFile::from(indexed)), Some(rendering))
}
None => (
indexer.index(&self.path, &self.source, self.key),
None,
None,
),
};
Prepared {
with_environment: cooked.is_some(),
rendering,
outcome: PreparedOutcome::Built {
summary,
stored: false,
cooked,
},
}
}
}
enum PreparedOutcome {
Unreadable,
Stored(FileSummary),
Built {
summary: FileSummary,
stored: bool,
cooked: Option<crate::CookedFile>,
},
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct StoreStats {
pub reused: usize,
pub rebuilt: usize,
pub unstored: usize,
pub stored_refinements_served: usize,
}
impl StoreStats {
pub fn hit_rate(&self) -> Option<f64> {
let total = self.reused + self.rebuilt;
(total > 0).then(|| self.reused as f64 / total as f64)
}
pub fn since(self, earlier: StoreStats) -> StoreStats {
StoreStats {
reused: self.reused.saturating_sub(earlier.reused),
rebuilt: self.rebuilt.saturating_sub(earlier.rebuilt),
unstored: self.unstored.saturating_sub(earlier.unstored),
stored_refinements_served: self
.stored_refinements_served
.saturating_sub(earlier.stored_refinements_served),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct IncludeBudget {
pub max_files: usize,
pub max_depth: usize,
}
impl Default for IncludeBudget {
fn default() -> Self {
IncludeBudget {
max_files: 4096,
max_depth: crate::MAX_INCLUDE_DEPTH,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct IncludeIndex {
pub indexed: Vec<PathBuf>,
pub unresolved: Vec<UnresolvedEdge>,
pub not_indexed: Vec<NotIndexed>,
pub re_read: usize,
pub stats: StoreStats,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UnresolvedEdge {
pub from: PathBuf,
pub spelling: String,
pub range: cpp_parser::SourceRange,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NotIndexed {
pub path: PathBuf,
pub reason: NotIndexedReason,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NotIndexedReason {
Budget,
Depth,
Unreadable,
}
impl SummaryStore<DiskFiles> {
pub fn open(root: impl Into<PathBuf>, config: CompilerConfig) -> SummaryStore<DiskFiles> {
SummaryStore::with_provider(root, config, DiskFiles)
}
}
impl<F: FileProvider> SummaryStore<F> {
pub fn with_provider(
root: impl Into<PathBuf>,
config: CompilerConfig,
files: F,
) -> SummaryStore<F> {
let root = root.into();
SummaryStore {
cache: root.join(crate::CACHE_DIRECTORY),
root,
config,
files,
index: ProjectIndex::new(),
stats: StoreStats::default(),
revision: std::sync::atomic::AtomicU64::new(0),
dead_macros: std::sync::Mutex::new(std::collections::HashMap::new()),
refined_marks: std::sync::Mutex::new(std::collections::HashSet::new()),
summary_database: std::sync::OnceLock::new(),
generation: 0,
include_searches: std::sync::Mutex::new(std::collections::HashMap::new()),
walk_cache: crate::index::walk_cache::WalkCache::new(),
directives: std::sync::Mutex::new(std::collections::HashMap::new()),
}
}
pub fn with_cache_directory(mut self, name: &str) -> Self {
if !name.is_empty() {
self.cache = self.root.join(name);
}
self
}
pub fn cache_directory(&self) -> &Path {
&self.cache
}
fn summary_database(&self) -> Option<std::sync::Arc<cache::cache_db::SummaryCache>> {
self.summary_database
.get_or_init(|| cache::cache_db::shared(&self.cache, crate::READING_FINGERPRINT))
.clone()
}
pub fn database(&self) -> Option<std::sync::Arc<cache::cache_db::SummaryCache>> {
self.summary_database()
}
pub fn begin_generation(&mut self) {
if let Some(database) = self.summary_database()
&& let Ok(generation) = database.begin_generation()
{
self.generation = generation;
}
}
pub fn cached_unit(&self, key: u64) -> Option<crate::RenderedUnit> {
self.summary_database()?.unit(key).ok().flatten()
}
pub fn cached_indexed_unit(&self, program: u64) -> Option<crate::IndexedUnit> {
self.summary_database()?
.indexed_unit(program)
.ok()
.flatten()
}
pub fn remember_indexed_unit(&self, program: u64, unit: &crate::IndexedUnit) {
if let Some(database) = self.summary_database() {
let _ = database.put_indexed_unit(program, unit);
}
}
pub fn remember_unit(&self, key: u64, unit: &crate::RenderedUnit) {
if let Some(database) = self.summary_database() {
let _ = database.put_unit(key, unit);
}
}
pub fn cached_unit_root(&self, root: &str) -> Option<u64> {
self.summary_database()?.unit_root(root).ok().flatten()
}
pub fn remember_unit_root(&self, root: &str, units_key: u64) {
if let Some(database) = self.summary_database() {
let _ = database.put_unit_root(root, units_key);
}
}
pub fn collect_the_cache(&mut self) -> Option<cache::cache_db::Collected> {
self.summary_database()
.and_then(|database| database.collect())
}
pub fn flush_summaries(&self) {
if let Some(database) = self.summary_database() {
database.flush();
}
}
pub fn summary_database_sizes(&self) -> Option<cache::cache_db::DatabaseSizes> {
self.summary_database()
.and_then(|database| database.sizes())
}
pub fn stats(&self) -> StoreStats {
self.stats
}
pub fn with_macros(mut self, macros: crate::include::graph::Marked) -> Self {
self.index = std::mem::take(&mut self.index).with_macros(macros);
self
}
pub fn index(&self) -> &ProjectIndex {
&self.index
}
pub fn include_search_count(&self) -> Option<usize> {
self.include_searches.lock().ok().map(|held| held.len())
}
pub(crate) fn contributions(&self) -> &crate::index::walk_cache::WalkCache {
&self.walk_cache
}
pub(crate) fn save_the_contributions(&self) -> usize {
let Some(database) = self.summary_database() else {
return 0;
};
let entries = self.contributions().saved();
if entries.is_empty() {
return 0;
}
let encoded: Vec<(u64, Vec<u8>)> = entries
.into_iter()
.map(|(key, contribution)| {
(
key,
crate::index::walk_cache_codec::encode_one(key, &contribution),
)
})
.collect();
database.put_contributions(&encoded).unwrap_or(0)
}
pub(crate) fn load_the_contributions(&self) -> usize {
let Some(database) = self.summary_database() else {
return 0;
};
let Ok(rows) = database.contributions() else {
return 0;
};
let mut entries = Vec::with_capacity(rows.len());
for (key, bytes) in rows {
if let Some(contribution) = crate::index::walk_cache_codec::decode_one(&bytes) {
entries.push((key, contribution));
}
}
let restored = entries.len();
self.contributions().restore(entries);
restored
}
pub fn directive_summary(&self, path: &Path) -> Option<std::sync::Arc<crate::FileSummary>> {
let source = self.files.read(path)?;
let hash = content_hash(&source);
let key = normalize_path(path, cfg!(windows));
if let Ok(known) = self.directives.lock()
&& let Some((held, summary)) = known.get(&key)
&& *held == hash
{
return Some(std::sync::Arc::clone(summary));
}
let summary = std::sync::Arc::new(
crate::index::FileIndexer::new(&self.files, &self.config)
.with_seed(self.index.macros())
.directive_summary(path, &source),
);
if let Ok(mut known) = self.directives.lock() {
if known.len() >= MAX_DIRECTIVE_READINGS {
known.clear();
}
known.insert(key, (hash, std::sync::Arc::clone(&summary)));
}
Some(summary)
}
pub fn index_the_rendering(
&mut self,
path: &Path,
rendered: &crate::preprocess::cooked::RenderedCooked,
environment: Option<&dyn cpp_parser::MacroFacts>,
) -> Option<&FileSummary> {
let source = self.files.read(path)?;
let mut key = SummaryKey::new(content_hash(&source), self.context_hash(path));
if let Some(facts) = environment {
key = key.with_environment(environment_hash(facts));
}
let indexer = FileIndexer::new(&self.files, &self.config).with_seed(self.index.macros());
let indexed = indexer.index_the_readings(path, &source, rendered, key);
let summary = indexed.summary;
self.index.insert(summary);
self.index.summary(path)
}
pub fn parsed_summary(&self, path: &Path) -> Option<crate::FileSummary> {
let source = self.files.read(path)?;
let key = crate::SummaryKey::new(crate::content_hash(&source), self.context_hash(path));
Some(
crate::index::FileIndexer::new(&self.files, &self.config)
.with_seed(self.index.macros())
.index(path, &source, key),
)
}
pub fn index_mut(&mut self) -> &mut ProjectIndex {
&mut self.index
}
pub fn config(&self) -> &CompilerConfig {
&self.config
}
pub fn get(&mut self, path: &Path) -> Option<&FileSummary> {
let prepared = self.prepare(path);
self.commit(path, prepared)
}
pub fn prepare(&self, path: &Path) -> Prepared {
self.prepare_inner(path, None, None)
}
pub fn prepare_with_the_timeline(
&self,
path: &Path,
unit: &crate::TranslationUnit,
) -> Prepared {
self.prepare_inner(path, None, Some(unit))
}
pub(crate) fn detached_read(
&self,
path: &Path,
timeline: Option<std::sync::Arc<crate::TranslationUnit>>,
) -> Option<DetachedRead<F>>
where
F: Clone,
{
let path = self
.index
.summary(path)
.map(|summary| summary.path.clone())
.unwrap_or_else(|| path.to_path_buf());
let source = self.files.read(&path)?;
let mut key = SummaryKey::new(content_hash(&source), self.context_hash(&path));
let timeline = timeline.filter(|unit| unit.environment_of(&path).is_some());
if let Some(view) = timeline
.as_ref()
.and_then(|unit| unit.environment_of(&path))
{
key = key.with_environment(environment_hash(&view));
}
Some(DetachedRead {
dead_macros: self.dead_macros_for(&path),
seed: self.index.macros().clone(),
path,
source,
config: self.config.clone(),
files: self.files.clone(),
key,
timeline,
})
}
fn prepare_telling(&self, path: &Path, includes: &mut dyn FnMut(&[PathBuf])) -> Prepared {
self.prepare_inner(path, Some(includes), None)
}
fn prepare_inner(
&self,
path: &Path,
mut includes: Option<&mut dyn FnMut(&[PathBuf])>,
timeline: Option<&crate::TranslationUnit>,
) -> Prepared {
let tracing = crate::probes::cooking();
let entered = std::time::Instant::now();
let Some(source) = ({ self.files.read(path) }) else {
return Prepared {
outcome: PreparedOutcome::Unreadable,
with_environment: false,
rendering: None,
};
};
let read = entered.elapsed();
let view = timeline.and_then(|unit| {
let spelled = self
.index
.summary(path)
.map(|summary| summary.path.clone())
.unwrap_or_else(|| path.to_path_buf());
unit.environment_of(&spelled)
});
let with_environment = view.is_some();
let mut key = SummaryKey::new(content_hash(&source), self.context_hash(path));
if let Some(view) = view.as_ref() {
key = key.with_environment(environment_hash(view));
}
{
let reading = std::time::Instant::now();
let stored = self
.summary_database()
.and_then(|database| database.summary(key))
.ok_or(());
if let Ok(stored) = stored
&& stored.key == key
{
let verifying = std::time::Instant::now();
let still_holds = self.resolution_still_holds(path, &stored);
if tracing {
note_a_prepared_file(
read,
reading.elapsed(),
verifying.elapsed(),
std::time::Duration::ZERO,
);
}
if still_holds {
if let Some(tell) = includes.as_mut() {
let targets: Vec<PathBuf> = stored
.includes
.iter()
.filter_map(|include| include.resolved.clone())
.collect();
tell(&targets);
}
return Prepared {
outcome: PreparedOutcome::Stored(stored),
with_environment,
rendering: None,
};
}
} else if tracing {
note_a_prepared_file(
read,
reading.elapsed(),
std::time::Duration::ZERO,
std::time::Duration::ZERO,
);
}
}
let scanned = match includes.is_some() {
true => {
let scanned = {
FileIndexer::new(&self.files, &self.config)
.with_seed(self.index.macros())
.scan_includes(path, &source)
};
if let Some(tell) = includes.as_mut() {
tell(&scanned.targets());
}
Some(scanned)
}
false => None,
};
let indexer = FileIndexer::new(&self.files, &self.config)
.with_seed(self.index.macros())
.with_scanned_includes_opt(scanned.as_ref())
.without_these_macros(&self.dead_macros_for(path));
let rendered = match (timeline, view) {
(Some(unit), Some(view)) => Some(self.the_rendering_of(path, &source, unit, view, key)),
_ => None,
};
let from_a_rendering = rendered.is_some();
let (summary, cooked) = match rendered {
Some((summary, cooked)) => (summary, Some(cooked)),
None => (indexer.index(path, &source, key), None),
};
let stored = !from_a_rendering && !has_unresolved_includes(&summary);
if stored {
if let Some(database) = self.summary_database() {
database.put(key, &summary);
database.note_file(path, key.content_hash, key.context_hash, 0, self.generation);
}
}
if tracing {
note_a_prepared_file(
read,
std::time::Duration::ZERO,
std::time::Duration::ZERO,
entered.elapsed(),
);
}
Prepared {
outcome: PreparedOutcome::Built {
summary,
stored,
cooked,
},
with_environment: from_a_rendering,
rendering: None,
}
}
fn the_rendering_of(
&self,
path: &Path,
source: &str,
unit: &crate::TranslationUnit,
view: crate::MacroView<'_>,
key: SummaryKey,
) -> (crate::FileSummary, crate::CookedFile) {
let seed = crate::macros::MacroTable::from_marked(self.index.macros());
let definitions = unit.definitions();
let (tokens, _) = cpp_parser::lex(source, &cpp_parser::LexerConfig::default());
let macros =
{ crate::preprocess::cooked::FileMacros::new(&view, &definitions, Some(&seed), true) };
let rendered = { crate::preprocess::cooked::cook_with(source, &tokens, ¯os).render() };
let directives = self.directive_summary(path);
let indexed = FileIndexer::new(&self.files, &self.config)
.with_seed(self.index.macros())
.without_these_macros(&self.dead_macros_for(path))
.with_directives(directives.as_deref())
.index_the_readings(path, source, &rendered, key);
let cooked = crate::CookedFile {
declarations: indexed.summary.declarations.clone(),
macros: indexed.summary.macros.clone(),
diagnostics: indexed.diagnostics,
unplaced: indexed.unplaced,
};
(indexed.summary, cooked)
}
pub fn commit(&mut self, path: &Path, prepared: Prepared) -> Option<&FileSummary> {
match prepared.outcome {
PreparedOutcome::Unreadable => None,
PreparedOutcome::Stored(stored) => {
self.stats.reused += 1;
self.index.insert_at(path, stored);
self.index.summary(path)
}
PreparedOutcome::Built {
summary,
stored,
cooked,
} => {
self.stats.rebuilt += 1;
if !stored {
self.stats.unstored += 1;
}
self.index.insert(summary);
if let Some(cooked) = cooked {
self.index.insert_cooked(path, cooked);
}
self.index.summary(path)
}
}
}
pub fn prepare_closure(
&self,
roots: &[PathBuf],
wanted: impl Fn(&Path) -> bool + Sync,
budget: usize,
) -> Vec<(PathBuf, Prepared)> {
use std::sync::{Condvar, Mutex};
let tracing = crate::probes::cooking();
let before = prepared_stages();
let started = std::time::Instant::now();
struct Frontier {
waiting: std::collections::VecDeque<PathBuf>,
seen: HashSet<String>,
in_flight: usize,
done: Vec<(PathBuf, Prepared)>,
panic: Option<Box<dyn std::any::Any + Send>>,
}
let mut frontier = Frontier {
waiting: std::collections::VecDeque::new(),
seen: HashSet::new(),
in_flight: 0,
done: Vec::new(),
panic: None,
};
for root in roots {
if frontier.seen.insert(normalize_path(root, cfg!(windows))) {
frontier.waiting.push_back(root.clone());
}
}
let budget = budget.max(frontier.waiting.len());
let shared = (Mutex::new(frontier), Condvar::new());
let lock = || {
shared
.0
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
};
let work = || {
loop {
let path = {
let mut frontier = lock();
loop {
if frontier.panic.is_some() {
break None;
}
if let Some(path) = frontier.waiting.pop_front() {
frontier.in_flight += 1;
break Some(path);
}
if frontier.in_flight == 0 {
break None;
}
frontier = shared
.1
.wait(frontier)
.unwrap_or_else(|poisoned| poisoned.into_inner());
}
};
let Some(path) = path else {
shared.1.notify_all();
return;
};
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
self.prepare_telling(&path, &mut |targets| {
let mut frontier = lock();
let mut added = false;
for target in targets {
if frontier.seen.len() >= budget {
break;
}
if !wanted(target)
|| !frontier.seen.insert(normalize_path(target, cfg!(windows)))
{
continue;
}
frontier.waiting.push_back(target.clone());
added = true;
}
drop(frontier);
if added {
shared.1.notify_all();
}
})
}));
let mut frontier = lock();
frontier.in_flight -= 1;
match outcome {
Ok(prepared) => frontier.done.push((path, prepared)),
Err(payload) => frontier.panic = Some(payload),
}
drop(frontier);
shared.1.notify_all();
}
};
let workers = std::thread::available_parallelism()
.map_or(1, |cores| cores.get())
.saturating_sub(1)
.clamp(1, PARALLEL_WORKERS);
if workers <= 1 {
work();
} else {
std::thread::scope(|scope| {
for _ in 1..workers {
let _ = std::thread::Builder::new()
.name("cppls-index".to_string())
.stack_size(WORKER_STACK)
.spawn_scoped(scope, work);
}
work();
});
}
let mut frontier = shared
.0
.into_inner()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(payload) = frontier.panic.take() {
std::panic::resume_unwind(payload);
}
if tracing {
let after = prepared_stages();
let millis = |slot: usize| (after[slot] - before[slot]) as f64 / 1e6;
let took = started.elapsed();
if took.as_millis() >= 40 {
eprintln!(
"prepare: {} file(s) in {} ms | text {:.0} ms, decode {:.0} ms, verify {:.0} ms, build+encode \
{:.0} ms (summed over workers)",
after[0] - before[0],
took.as_millis(),
millis(1),
millis(2),
millis(3),
millis(4)
);
}
}
frontier.done
}
pub fn prepare_many(&self, paths: &[PathBuf]) -> Vec<Prepared> {
parallel_map(paths, |path| self.prepare(path))
}
pub fn index_includes_from(&mut self, entry: &Path, budget: IncludeBudget) -> IncludeIndex {
let before = self.stats;
let mut outcome = IncludeIndex::default();
let mut seen: HashSet<String> = HashSet::new();
let mut pending: Vec<(PathBuf, usize)> = vec![(entry.to_path_buf(), 0)];
while let Some((path, depth)) = pending.pop() {
if !seen.insert(normalize_path(&path, cfg!(windows))) {
continue;
}
if outcome.indexed.len() >= budget.max_files {
outcome.not_indexed.push(NotIndexed {
path,
reason: NotIndexedReason::Budget,
});
continue;
}
if depth > budget.max_depth {
outcome.not_indexed.push(NotIndexed {
path,
reason: NotIndexedReason::Depth,
});
continue;
}
let Some(summary) = self.get(&path) else {
outcome.not_indexed.push(NotIndexed {
path,
reason: NotIndexedReason::Unreadable,
});
continue;
};
let includes: Vec<IncludeFact> = summary.includes.clone();
outcome.indexed.push(path.clone());
for include in includes.into_iter().rev() {
match include.resolved {
Some(target) => pending.push((target, depth + 1)),
None => outcome.unresolved.push(UnresolvedEdge {
from: path.clone(),
spelling: include.spelling,
range: include.range,
}),
}
}
}
let truncated = !outcome.not_indexed.is_empty();
let indexed = outcome.indexed.clone();
let plan = self.prepare_the_re_read(&indexed, &[], truncated, false);
outcome.re_read = self.commit_the_re_read(plan);
outcome.stats = self.stats.since(before);
outcome
}
pub fn re_read_where_a_body_decides(
&mut self,
files: &[PathBuf],
units: &[std::sync::Arc<crate::TranslationUnit>],
) -> usize {
let plan = self.prepare_the_re_read(files, units, false, true);
self.commit_the_re_read(plan)
}
pub fn commit_the_re_read(&mut self, plan: ReRead) -> usize {
for (path, cooked) in plan.cooked {
self.put_the_cooked_reading(&path, &cooked);
self.index.insert_cooked(&path, cooked);
}
let mut re_read = 0;
for (mut rebuilt, stored) in plan.rebuilt {
let Some(current) = self.index.summary(&rebuilt.path).map(|summary| summary.key) else {
continue;
};
if current.content_hash != rebuilt.key.content_hash
|| current.context_hash != rebuilt.key.context_hash
{
continue;
}
rebuilt.key = current;
self.commit_reread(rebuilt, stored);
re_read += 1;
}
re_read
}
pub fn the_closure_fingerprint(&self, path: &Path) -> Option<u64> {
const THE_WIDEST_CLOSURE: usize = 8192;
const UNRESOLVED_INCLUDE: u64 = 0x8f3a_2c11_5e6b_9d07;
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut pending: std::collections::VecDeque<PathBuf> = std::collections::VecDeque::new();
let mut parts: Vec<u64> = Vec::new();
pending.push_back(path.to_path_buf());
while let Some(next) = pending.pop_front() {
let key = normalize_path(&next, cfg!(windows));
if !seen.insert(key.clone()) {
continue;
}
if seen.len() > THE_WIDEST_CLOSURE {
return None;
}
let Some(summary) = self.index.summary(std::path::Path::new(&key)) else {
return None;
};
let mut bytes = Vec::with_capacity(16 + key.len());
bytes.extend_from_slice(key.as_bytes());
bytes.extend_from_slice(&summary.key.content_hash.to_le_bytes());
parts.push(fnv1a64(&bytes));
for include in &summary.includes {
match include.resolved.as_ref() {
Some(resolved) => pending.push_back(resolved.clone()),
None => parts.push(fnv1a64(include.spelling.as_bytes()) ^ UNRESOLVED_INCLUDE),
}
}
}
parts.sort_unstable();
let mut bytes = Vec::with_capacity(parts.len() * 8);
for part in parts {
bytes.extend_from_slice(&part.to_le_bytes());
}
Some(fnv1a64(&bytes))
}
fn the_record_environment(&self, path: &Path) -> Option<u64> {
self.the_indexed_environment(path)
.map(|environment| environment.fingerprint())
}
fn mark_the_reading_as_refined(&self, path: &Path) {
let Some(summary) = self.index.summary(path) else {
return;
};
let Some(fingerprint) = self.the_closure_fingerprint(path) else {
return;
};
let mut marked = summary.clone();
marked.key = SummaryKey::new(marked.key.content_hash, marked.key.context_hash)
.with_environment(fingerprint);
store_it(&self.summary_database(), path, &marked, self.generation);
if let Ok(mut marks) = self.refined_marks.lock() {
marks.insert(normalize_path(path, cfg!(windows)));
}
}
pub fn is_marked_refined(&self, path: &Path) -> bool {
self.refined_marks
.lock()
.is_ok_and(|marks| marks.contains(&normalize_path(path, cfg!(windows))))
}
pub fn the_stored_refinement(&self, path: &Path) -> Option<crate::FileSummary> {
match self.the_stored_refinement_reason(path) {
Ok(summary) => Some(summary),
Err(reason) => {
if crate::probes::cooking() {
eprintln!("refinement: {} skipped: {reason}", path.display());
}
None
}
}
}
fn the_stored_refinement_reason(
&self,
path: &Path,
) -> Result<crate::FileSummary, &'static str> {
let Some(source) = self.files.read(path) else {
return Err("no text");
};
let Some(fingerprint) = self.the_closure_fingerprint(path) else {
return Err("no fingerprint");
};
let key = SummaryKey::new(content_hash(&source), self.context_hash(path))
.with_environment(fingerprint);
let Some(database) = self.summary_database() else {
return Err("no database");
};
let Some(stored) = database.summary(key) else {
return Err("not stored");
};
if stored.key != key {
return Err("key moved");
}
if !self.resolution_still_holds(path, &stored) {
return Err("resolution moved");
}
Ok(stored)
}
pub fn apply_a_stored_refinement(&mut self, summary: crate::FileSummary) {
self.commit_reread(summary, true);
}
pub fn put_the_cooked_reading_with_rendering(
&self,
path: &Path,
reading: &CookedFile,
rendered: &crate::preprocess::cooked::RenderedCooked,
) {
let Some(source) = self.files.read(path) else {
return;
};
let Some(fingerprint) = self.the_record_environment(path) else {
return;
};
let key = CacheKey {
content: content_hash(&source),
config: self.context_hash(path),
env: fingerprint,
};
let Some(database) = self.summary_database() else {
return;
};
let bytes = crate::summary_codec::encode_the_cooked_record(reading, Some(rendered));
database.put_cooked_reading(key, &bytes);
}
pub fn the_stored_cooked_reading_with_rendering(
&self,
path: &Path,
) -> Option<(CookedFile, crate::preprocess::cooked::RenderedCooked)> {
self.the_stored_cook_reason(path).ok()
}
pub fn the_stored_cook_reason(
&self,
path: &Path,
) -> Result<(CookedFile, crate::preprocess::cooked::RenderedCooked), &'static str> {
let source = self.files.read(path).ok_or("no text")?;
let fingerprint = self.the_record_environment(path).ok_or("no closure")?;
let key = CacheKey {
content: content_hash(&source),
config: self.context_hash(path),
env: fingerprint,
};
let database = self.summary_database().ok_or("no database")?;
let bytes = database.cooked_reading(key).ok_or("not stored")?;
let (cooked, rendered) =
crate::summary_codec::decode_the_cooked_record(&bytes).map_err(|_| "undecodable")?;
let rendered = rendered.ok_or("no rendering")?;
Ok((cooked, rendered))
}
pub fn put_the_cooked_reading(&self, path: &Path, reading: &CookedFile) {
let Some(source) = self.files.read(path) else {
return;
};
let Some(fingerprint) = self.the_record_environment(path) else {
return;
};
let key = CacheKey {
content: content_hash(&source),
config: self.context_hash(path),
env: fingerprint,
};
let Some(database) = self.summary_database() else {
return;
};
const KIND_AT: usize = 8 + 4;
if database
.cooked_reading(key)
.is_some_and(|existing| existing.get(KIND_AT) == Some(&1))
{
return;
}
let bytes = crate::summary_codec::encode_the_cooked_reading(reading);
database.put_cooked_reading(key, &bytes);
}
pub fn the_stored_cooked_reading(&self, path: &Path) -> Option<CookedFile> {
let source = self.files.read(path)?;
let fingerprint = self.the_record_environment(path)?;
let key = CacheKey {
content: content_hash(&source),
config: self.context_hash(path),
env: fingerprint,
};
let database = self.summary_database()?;
let bytes = database.cooked_reading(key)?;
crate::summary_codec::decode_the_cooked_reading(&bytes).ok()
}
fn the_stored_refinement_mark(&self, path: &Path) -> bool {
let Some(summary) = self.index.summary(path) else {
return false;
};
let Some(fingerprint) = self.the_closure_fingerprint(path) else {
return false;
};
let key = crate::cache::SummaryKey::new(summary.key.content_hash, summary.key.context_hash)
.with_environment(fingerprint);
let Some(database) = self.summary_database() else {
return false;
};
database.summary(key).is_some()
}
pub fn take_the_stored_refinements(&mut self, files: Vec<PathBuf>) -> ServedRefinements {
let mut still_to_do = Vec::new();
let mut served_paths = Vec::new();
let mut served = 0usize;
for path in files {
let Some(cooked) = self.the_stored_cooked_reading(&path) else {
if self.the_stored_refinement_mark(&path) {
if let Ok(mut marks) = self.refined_marks.lock() {
marks.insert(normalize_path(&path, cfg!(windows)));
}
served += 1;
continue;
}
still_to_do.push(path);
continue;
};
self.index.insert_cooked(&path, cooked);
if let Ok(mut marks) = self.refined_marks.lock() {
marks.insert(normalize_path(&path, cfg!(windows)));
}
served_paths.push(path);
served += 1;
}
self.stats.stored_refinements_served += served;
if crate::probes::cooking() && !still_to_do.is_empty() {
eprintln!(
"re-read: {served} stored refinement(s) served, {} rebuilt (nothing stored for them)",
still_to_do.len()
);
}
ServedRefinements {
still_to_do,
served: served_paths,
}
}
pub fn prepare_the_re_read(
&self,
indexed: &[PathBuf],
units: &[std::sync::Arc<crate::TranslationUnit>],
truncated: bool,
walk_its_own: bool,
) -> ReRead {
let _ = truncated;
let tracing = crate::probes::cooking();
let started = std::time::Instant::now();
if let Ok(mut dead) = self.dead_macros.lock() {
for unit in units {
for (path, offsets) in &unit.dead_macros {
let entry = dead.entry(normalize_path(path, cfg!(windows))).or_default();
entry.extend(offsets.iter().copied());
entry.sort_unstable();
entry.dedup();
}
}
}
let mut sources: std::collections::HashMap<String, String> =
std::collections::HashMap::new();
let eager = units.is_empty() && !walk_its_own;
{
if eager {
for summary in self.index.summaries() {
self.read_into(&mut sources, &summary.path);
}
} else {
for path in indexed {
self.read_into(&mut sources, path);
}
}
}
let seed = crate::macros::MacroTable::from_marked(self.index.macros());
let unit_definitions: Vec<crate::UnitDefinitions<'_>> =
units.iter().map(|unit| unit.definitions()).collect();
let environments: Vec<(
&std::sync::Arc<crate::TranslationUnit>,
crate::UnitDefinitions<'_>,
)> = units.iter().zip(unit_definitions).collect();
let mentioned = words_mentioned_by(&sources, indexed);
let wanted = |name: &str| mentioned.contains(name);
let closure_started = std::time::Instant::now();
let in_the_closure = self.the_closure_of(indexed);
if tracing {
eprintln!(
"re-read: closure of {} file(s) is {} file(s) in {:?}",
indexed.len(),
in_the_closure.len(),
closure_started.elapsed()
);
}
let mut candidates: Vec<&FileSummary> = if eager {
self.index.summaries().collect()
} else {
self.index
.files_defining_any_macro(mentioned.iter().map(String::as_str))
.into_iter()
.filter(|summary| {
in_the_closure.contains(&normalize_path(&summary.path, cfg!(windows)))
})
.collect()
};
let uncovered_candidates = candidates
.iter()
.filter(|summary| bodies_of(&environments, &summary.path).is_none())
.count();
let uncovered = uncovered_candidates > 0;
if !eager {
if !environments.is_empty() {
candidates.retain(|summary| bodies_of(&environments, &summary.path).is_some());
}
for path in indexed {
if units.iter().any(|unit| unit.environment_of(path).is_some()) {
continue;
}
self.read_into(&mut sources, path);
for reached in self.includes_reached_from(path) {
self.read_into(&mut sources, &reached);
}
}
}
let uncovered = uncovered || uncovered_candidates > 0;
let mut bodied: Vec<String> = Vec::new();
let mut is_bodied: std::collections::HashSet<&str> = std::collections::HashSet::new();
let mut definitions = crate::summary::MacroDefinitions::default();
for summary in candidates.iter().copied() {
let key = normalize_path(&summary.path, cfg!(windows));
if !summary
.macros
.iter()
.any(|fact| fact.kind.is_definition() && wanted(&fact.name))
{
continue;
}
let Some(source) = sources.get(&key) else {
continue;
};
let mut environment: Option<cpp_parser::MacroEnvironment> = None;
let bodies = bodies_of(&environments, &summary.path);
for fact in &summary.macros {
if !fact.kind.is_definition() || is_bodied.contains(fact.name.as_str()) {
continue;
}
let Some(range) = fact.body_range else {
continue;
};
let Some(body) = source.get(range.start_offset..range.start_offset + range.length)
else {
continue;
};
let used = match the_plain_shape(body) {
APlainShape::Used => true,
APlainShape::Settled => false,
APlainShape::Unplaceable => match &bodies {
Some((view, definitions)) => {
let bodies = crate::preprocess::cooked::FileMacros::new(
view,
*definitions,
Some(&seed),
true,
);
cpp_parser::shape_of_a_body_at(body, &bodies, range.start_offset)
.a_reading_uses_this()
}
None => {
let environment = environment.get_or_insert_with(|| {
let evidence = crate::summary::macros_from_the_closure_with_bodies(
summary,
|wanted| {
let key = normalize_path(wanted, cfg!(windows));
Some((
self.index.summary(std::path::Path::new(&key))?,
sources.get(&key)?.as_str(),
))
},
self.index.macros(),
&mut definitions,
);
cpp_parser::MacroEnvironment::from_included_macros(evidence.macros)
.with_bodies_in_force(evidence.conditional_bodies)
});
cpp_parser::shape_of_a_body_at(body, environment, range.start_offset)
.a_reading_uses_this()
}
},
};
if used {
is_bodied.insert(&fact.name);
bodied.push(fact.name.to_string());
}
}
}
let guarded: std::collections::HashSet<&Path> = indexed
.iter()
.filter(|path| {
self.index.summary(path).is_some_and(|summary| {
summary.declarations.iter().any(|fact| {
fact.split_by_a_directive
|| !matches!(fact.guard, crate::summary::FactGuard::Unconditional)
})
})
})
.map(PathBuf::as_path)
.collect();
if bodied.is_empty() && guarded.is_empty() {
if tracing {
eprintln!(
"re-read: eager {eager}, {} source(s), {} word(s) mentioned, {} candidate file(s), \
uncovered {uncovered}, nothing bodied — {} ms",
sources.len(),
mentioned.len(),
candidates.len(),
started.elapsed().as_millis()
);
}
let marking = std::time::Instant::now();
for path in indexed {
self.mark_the_reading_as_refined(path);
}
if tracing {
eprintln!(
"re-read: marked {} file(s) in {:?}",
indexed.len(),
marking.elapsed()
);
}
return ReRead {
rebuilt: Vec::new(),
cooked: Vec::new(),
not_yet: Vec::new(),
};
}
let bodied_names: std::collections::HashSet<&str> =
bodied.iter().map(String::as_str).collect();
let mut rebuilt: Vec<(FileSummary, bool)> = Vec::new();
let mut cooked: Vec<(PathBuf, crate::CookedFile)> = Vec::new();
let not_yet: Vec<PathBuf> = Vec::new();
let mut through_a_timeline: Vec<(&PathBuf, &String)> = Vec::new();
for path in indexed {
let Some(source) = sources.get(&normalize_path(path, cfg!(windows))) else {
continue;
};
let mentioned = { mentions_one_of(source, &bodied_names) };
let decided_about = !self.dead_macros_for(path).is_empty();
if !mentioned && !decided_about && !guarded.contains(path.as_path()) {
self.mark_the_reading_as_refined(path);
continue;
}
if tracing {
eprintln!("re-read candidate: {}", path.display());
}
if units.iter().any(|unit| unit.environment_of(path).is_some()) {
through_a_timeline.push((path, source));
continue;
}
if !walk_its_own {
let Some(fingerprint) = self.the_closure_fingerprint(path) else {
continue;
};
let key = SummaryKey::new(content_hash(source), self.context_hash(path))
.with_environment(fingerprint);
let indexer =
FileIndexer::new(&self.files, &self.config).with_seed(self.index.macros());
let Some(environment) = self.closure_environment(path, &sources) else {
continue;
};
let made = indexer
.with_macro_bodies(&environment)
.index(path, source, key);
store_it(&self.summary_database(), path, &made, self.generation);
rebuilt.push((made, true));
continue;
}
let Some(fingerprint) = self.the_closure_fingerprint(path) else {
continue;
};
let key = SummaryKey::new(content_hash(source), self.context_hash(path))
.with_environment(fingerprint);
let indexer =
FileIndexer::new(&self.files, &self.config).with_seed(self.index.macros());
if let Some(unit) = self.a_timeline_of_its_own(path, &sources)
&& let Some(view) = unit.environment_of(path)
{
let definitions = unit.definitions();
let (tokens, _) = cpp_parser::lex(source, &cpp_parser::LexerConfig::default());
let macros = crate::preprocess::cooked::FileMacros::new(
&view,
&definitions,
Some(&seed),
true,
);
let rendered =
{ crate::preprocess::cooked::cook_with(source, &tokens, ¯os).render() };
let dead = self.dead_macros_for(path);
let indexed = indexer
.without_these_macros(&dead)
.index_the_readings(path, source, &rendered, key);
cooked.push((path.to_path_buf(), indexed.into()));
if let Some(raw) = self.index.summary(path) {
rebuilt.push((raw.clone(), false));
} else {
self.mark_the_reading_as_refined(path);
}
if tracing {
eprintln!("re-read: {} through a walk of its own", path.display());
}
continue;
}
if tracing {
eprintln!("re-read: {} has no walk — composed", path.display());
}
let Some(environment) = self.the_indexed_environment(path) else {
continue;
};
let made = indexer
.with_macro_bodies(&environment)
.index(path, source, key);
let _ = made;
if let Some(raw) = self.index.summary(path) {
rebuilt.push((raw.clone(), false));
}
continue;
}
let rebuilt_through_a_timeline = parallel_map(&through_a_timeline, |(path, source)| {
let (view, definitions) = environments
.iter()
.find_map(|(unit, definitions)| Some((unit.environment_of(path)?, definitions)))?;
let key = SummaryKey::new(content_hash(source), self.context_hash(path));
let (tokens, _) = cpp_parser::lex(source, &cpp_parser::LexerConfig::default());
let macros =
crate::preprocess::cooked::FileMacros::new(&view, definitions, Some(&seed), true);
let rendered =
{ crate::preprocess::cooked::cook_with(source, &tokens, ¯os).render() };
let dead = self.dead_macros_for(path);
let rebuilt = FileIndexer::new(&self.files, &self.config)
.with_seed(self.index.macros())
.without_these_macros(&dead)
.index_the_readings(path, source, &rendered, key)
.summary;
let stored = false;
if tracing {
eprintln!("re-read: {} through its timeline", path.display());
}
Some((rebuilt, stored))
});
rebuilt.extend(rebuilt_through_a_timeline.into_iter().flatten());
if tracing {
eprintln!(
"re-read: eager {eager}, {} source(s), {} word(s) mentioned, {} candidate file(s), \
uncovered {uncovered}, {} bodied name(s) — {} ms",
sources.len(),
mentioned.len(),
candidates.len(),
bodied.len(),
started.elapsed().as_millis()
);
}
if rebuilt.iter().any(|(_, stored)| *stored)
&& let Some(database) = self.summary_database()
{
database.flush();
}
ReRead {
rebuilt,
cooked,
not_yet,
}
}
fn commit_reread(&mut self, rebuilt: FileSummary, stored: bool) {
self.stats.rebuilt += 1;
if !stored {
self.stats.unstored += 1;
}
self.index.insert(rebuilt);
}
fn dead_macros_for(&self, path: &Path) -> Vec<usize> {
let wanted = normalize_path(path, cfg!(windows));
let Ok(dead) = self.dead_macros.lock() else {
return Vec::new();
};
dead.get(&wanted).cloned().unwrap_or_default()
}
pub fn dead_macro_decisions(&self) -> (usize, usize) {
let Ok(dead) = self.dead_macros.lock() else {
return (0, 0);
};
(dead.len(), dead.values().map(|offsets| offsets.len()).sum())
}
fn read_into(&self, sources: &mut std::collections::HashMap<String, String>, path: &Path) {
let key = normalize_path(path, cfg!(windows));
if sources.contains_key(&key) {
return;
}
if let Some(text) = self.files.read(path) {
sources.insert(key, text);
}
}
fn the_closure_of(&self, roots: &[PathBuf]) -> std::collections::HashSet<String> {
let mut closure: std::collections::HashSet<String> = std::collections::HashSet::new();
for path in roots {
closure.insert(normalize_path(path, cfg!(windows)));
for reached in self.includes_reached_from(path) {
closure.insert(normalize_path(&reached, cfg!(windows)));
}
}
closure
}
fn includes_reached_from(&self, root: &Path) -> Vec<PathBuf> {
let mut reached: Vec<PathBuf> = Vec::new();
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut pending: Vec<PathBuf> = vec![root.to_path_buf()];
while let Some(path) = pending.pop() {
if !seen.insert(normalize_path(&path, cfg!(windows))) {
continue;
}
let Some(summary) = self.index.summary(&path) else {
continue;
};
for include in &summary.includes {
if let Some(resolved) = &include.resolved {
reached.push(resolved.clone());
pending.push(resolved.clone());
}
}
}
reached
}
fn closure_environment(
&self,
path: &Path,
sources: &std::collections::HashMap<String, String>,
) -> Option<cpp_parser::MacroEnvironment> {
let summary = self.index.summary(path)?;
let mut definitions = crate::summary::MacroDefinitions::default();
let evidence = crate::summary::macros_from_the_closure_with_bodies(
summary,
|wanted| {
Some((
self.index.summary(wanted)?,
sources
.get(&normalize_path(wanted, cfg!(windows)))
.map(String::as_str)
.unwrap_or(""),
))
},
self.index.macros(),
&mut definitions,
);
Some(
cpp_parser::MacroEnvironment::from_included_macros(evidence.macros)
.with_bodies_in_force(evidence.conditional_bodies),
)
}
pub(crate) fn the_indexed_environment(
&self,
path: &Path,
) -> Option<cpp_parser::MacroEnvironment> {
enum Step {
Enter { file: PathBuf, from: usize },
Fact {
file: PathBuf,
from: usize,
index: usize,
},
}
let root = normalize_path(path, cfg!(windows));
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut order: Vec<(usize, PathBuf, usize)> = Vec::new();
let mut pending: Vec<Step> = vec![Step::Enter {
file: path.to_path_buf(),
from: 0,
}];
while let Some(step) = pending.pop() {
let Step::Enter { file, from } = step else {
let Step::Fact { file, from, index } = step else {
unreachable!("the only two steps are `Enter` and `Fact`")
};
order.push((from, file, index));
continue;
};
let key = normalize_path(&file, cfg!(windows));
if !seen.insert(key.clone()) {
continue;
}
let Some(summary) = self.index.summary(std::path::Path::new(&key)) else {
continue;
};
let is_the_root = key == root;
let mut events: Vec<(usize, Step)> =
Vec::with_capacity(summary.includes.len() + summary.macros.len());
for include in &summary.includes {
if let Some(resolved) = &include.resolved {
let child_from = if is_the_root {
include.range.start_offset
} else {
from
};
events.push((
include.range.start_offset,
Step::Enter {
file: resolved.clone(),
from: child_from,
},
));
}
}
for (index, fact) in summary.macros.iter().enumerate() {
events.push((
fact.range.start_offset,
Step::Fact {
file: file.clone(),
from,
index,
},
));
}
events.sort_by_key(|(offset, _)| *offset);
for (_, event) in events.into_iter().rev() {
pending.push(event);
}
}
let mut table = crate::preprocess::macros::MacroTable::from_marked(self.index.macros());
let mut parsed = crate::cooked::ParsedDefinitions::new();
let mut decisions: std::collections::HashMap<
(usize, String),
(bool, cpp_parser::MacroBody, Option<String>, Option<String>),
> = std::collections::HashMap::new();
let mut texts: std::collections::HashMap<String, String> = std::collections::HashMap::new();
let mut answers_per_file: std::collections::HashMap<
String,
std::collections::HashMap<(u32, usize), Option<bool>>,
> = std::collections::HashMap::new();
for (from, file, index) in order.iter() {
let key = normalize_path(file, cfg!(windows));
if key == root {
continue;
}
let Some(summary) = self.index.summary(std::path::Path::new(&key)) else {
continue;
};
let Some(fact) = summary.macros.get(*index) else {
continue;
};
let answers = answers_per_file.entry(key.clone()).or_default();
let visibility = a_facts_visibility(&summary.guards, fact, &table, answers);
let is_a_definition = fact.kind.is_definition();
let taken = match visibility {
crate::Visibility::Active => true,
crate::Visibility::Inactive => false,
crate::Visibility::Unknown => !is_a_definition,
};
if !taken {
continue;
}
let place = (*from, fact.name.to_string());
if !is_a_definition {
table.undefine(&fact.name, *from);
decisions.insert(place, (fact.function_like, fact.body, None, None));
continue;
}
let Some(range) = fact.body_range else {
parsed.define_in(&mut table, &fact.name, None, "", *from);
decisions.insert(
place,
(fact.function_like, fact.body, Some(String::new()), None),
);
continue;
};
if !texts.contains_key(&key) {
let Some(text) = self.files.read(file) else {
continue;
};
texts.insert(key.clone(), text);
}
let Some(text) = texts.get(&key) else {
continue;
};
let Some(body) = text.get(range.start_offset..range.start_offset + range.length) else {
continue;
};
let parameters = fact.function_like.then(|| {
text.get(fact.range.start_offset + fact.range.length..range.start_offset)
.map(str::trim)
.unwrap_or("()")
.to_string()
});
parsed.define_in(&mut table, &fact.name, parameters.as_deref(), body, *from);
decisions.insert(
place,
(
fact.function_like,
fact.body,
Some(body.to_string()),
parameters,
),
);
}
let mut entries: Vec<cpp_parser::IncludedMacro> = decisions
.into_iter()
.map(
|((from, name), (function_like, shape, body, parameters))| match body {
Some(body) => cpp_parser::IncludedMacro::defined_with_body_and_parameters(
from,
&name,
function_like,
shape,
Some(&body),
parameters.as_deref(),
),
None => cpp_parser::IncludedMacro {
from_offset: from,
name: std::sync::Arc::from(name.as_str()),
definition: None,
body_text: None,
parameters: None,
},
},
)
.collect();
entries.sort_by(|left, right| {
left.from_offset
.cmp(&right.from_offset)
.then_with(|| left.name.cmp(&right.name))
});
if crate::probes::cooking() {
Self::trace_the_composed_environment(
path,
&order,
&entries,
&table,
&["JSON_TRY", "JSON_CATCH", "JSON_NOEXCEPTION", "_CPPUNWIND"],
);
}
Some(cpp_parser::MacroEnvironment::from_included_macros(entries))
}
fn trace_the_composed_environment(
path: &Path,
order: &[(usize, PathBuf, usize)],
entries: &[cpp_parser::IncludedMacro],
table: &crate::preprocess::macros::MacroTable,
watched: &[&str],
) {
eprintln!(
"[environment] {}: {} file(s) in the closure, {} decision(s), {} in the table",
path.display(),
order.len(),
entries.len(),
table.len()
);
for name in watched {
let own: Vec<String> = entries
.iter()
.filter(|entry| entry.name.as_ref() == *name)
.map(|entry| {
format!(
"{} at {}",
if entry.definition.is_some() {
"defined"
} else {
"undefined"
},
entry.from_offset
)
})
.collect();
eprintln!(
"[environment] {name}: in the table {}, in the timeline {own:?}",
table.get(name).is_some()
);
}
}
#[allow(dead_code)]
fn a_timeline_of_its_own(
&self,
path: &Path,
sources: &std::collections::HashMap<String, String>,
) -> Option<std::sync::Arc<crate::TranslationUnit>> {
let root = self.index.summary(path)?;
let mut texts: std::collections::HashMap<String, String> = std::collections::HashMap::new();
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut queue: std::collections::VecDeque<std::path::PathBuf> =
std::collections::VecDeque::new();
queue.push_back(root.path.clone());
let mut complete = true;
while let Some(file) = queue.pop_front() {
let key = normalize_path(&file, cfg!(windows));
if !seen.insert(key.clone()) {
continue;
}
let Some(text) = sources
.get(&key)
.cloned()
.or_else(|| self.files.read(&file))
else {
complete = false;
continue;
};
match self.index.summary(&file) {
Some(summary) => queue.extend(
summary
.includes
.iter()
.filter_map(|include| include.resolved.clone()),
),
None => complete = false,
}
texts.insert(key, text);
}
if !complete {
return None;
}
let mut definitions = crate::summary::MacroDefinitions::default();
let unit = crate::TranslationUnit::walk(
root,
|wanted| {
let summary = self.index.summary(wanted)?;
let key = normalize_path(wanted, cfg!(windows));
let text = texts.get(&key)?;
Some((summary, text.as_str()))
},
self.index.macros(),
&mut definitions,
false,
self.contributions(),
);
Some(std::sync::Arc::new(unit))
}
pub fn revision(&self) -> u64 {
self.revision.load(std::sync::atomic::Ordering::Relaxed)
}
fn a_summary_was_dropped(&self) {
self.revision
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
self.contributions().clear();
}
pub fn forget(&mut self, path: &Path) -> bool {
self.a_summary_was_dropped();
let key = normalize_path(path, cfg!(windows));
self.refined_marks
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&key);
self.dead_macros
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&key);
self.index.forget(path)
}
fn resolution_still_holds(&self, path: &Path, summary: &FileSummary) -> bool {
if summary.includes.is_empty() {
return true;
}
let directory = path.parent().unwrap_or(Path::new("."));
summary
.includes
.iter()
.all(|fact| self.where_it_resolves_now(fact, directory) == fact.resolved)
}
fn where_it_resolves_now(&self, fact: &IncludeFact, directory: &Path) -> Option<PathBuf> {
let key = (
fact.form,
fact.is_next,
Box::from(fact.spelling.as_str()),
match fact.form {
IncludeForm::Angle => PathBuf::new(),
IncludeForm::Quote | IncludeForm::Macro => directory.to_path_buf(),
},
);
if let Ok(searches) = self.include_searches.lock()
&& let Some(known) = searches.get(&key)
{
return known.clone();
}
let stored_key = self.include_search_key(&key);
if let Some(known) = self.stored_include_search(&stored_key) {
if let Ok(mut searches) = self.include_searches.lock() {
searches.insert(key, known.clone());
}
return known;
}
let resolver = crate::include::IncludeResolver::new(&self.files, &self.config);
let mut interner = crate::file::paths::PathInterner::new(cfg!(windows));
let resolved = resolver
.resolve(&fact.as_include(), directory, None, &mut interner)
.resolved()
.map(|found| found.path.clone());
if let Ok(mut searches) = self.include_searches.lock() {
if searches.len() >= MAX_INCLUDE_SEARCHES {
searches.clear();
}
searches.insert(key, resolved.clone());
}
self.remember_include_search(&stored_key, resolved.as_deref());
resolved
}
fn include_search_key(&self, memory: &(IncludeForm, bool, Box<str>, PathBuf)) -> String {
let (form, is_next, spelling, directory) = memory;
let paths: Vec<String> = self
.config
.include_paths
.iter()
.map(|path| format!("{}:{}", path.directory.to_string_lossy(), path.is_system))
.collect();
let folded = format!(
"{form:?}|{is_next}|{spelling}|{directory:?}|{paths:?}|{:?}",
self.config.dialect()
);
crate::cache::content_hash(&folded).to_string()
}
fn stored_include_search(&self, key: &str) -> Option<Option<PathBuf>> {
let database = self.summary_database()?;
let answer = database.include_search(key).ok().flatten()?;
Some((!answer.is_empty()).then(|| PathBuf::from(answer)))
}
fn remember_include_search(&self, key: &str, resolved: Option<&Path>) {
let Some(database) = self.summary_database() else {
return;
};
let answer = resolved
.map(|path| path.to_string_lossy().to_string())
.unwrap_or_default();
let _ = database.put_include_search(key, &answer);
}
pub fn invalidate(&self, path: &Path) -> Vec<PathBuf> {
self.a_summary_was_dropped();
let mut work = Vec::new();
let mut visited: HashSet<String> = HashSet::new();
let mut pending = vec![path.to_path_buf()];
while let Some(current) = pending.pop() {
let key = normalize_path(¤t, cfg!(windows));
if !visited.insert(key) {
continue;
}
work.push(current.clone());
for includer in self.index.includers_of(¤t) {
pending.push(includer);
}
}
work
}
pub fn context_hash(&self, path: &Path) -> u64 {
let mut bytes = Vec::new();
for include_path in &self.config.include_paths {
let directory = self
.config
.resolve_against_working_directory(&include_path.directory);
bytes.extend_from_slice(normalize_path(&directory, cfg!(windows)).as_bytes());
bytes.push(u8::from(include_path.is_system));
bytes.push(0);
}
bytes.push(b'|');
for define in &self.config.defines {
bytes.extend_from_slice(define.name.as_bytes());
bytes.push(b'=');
if let Some(value) = &define.value {
bytes.extend_from_slice(value.as_bytes());
}
bytes.push(0);
}
bytes.push(b'|');
for undefine in &self.config.undefines {
bytes.extend_from_slice(undefine.as_bytes());
bytes.push(0);
}
bytes.push(b'|');
for text in [&self.config.standard, &self.config.target] {
if let Some(text) = text {
bytes.extend_from_slice(text.as_bytes());
}
bytes.push(0);
}
bytes.push(b'|');
bytes.push(match self.config.dialect() {
Dialect::Gnu => b'g',
Dialect::Msvc => b'm',
});
bytes.push(if self.config.cuda() { b'c' } else { b'-' });
bytes.push(0);
bytes.push(u8::from(self.config.fuzzy_includes));
bytes.push(0);
bytes.push(b'c');
bytes.push(u8::from(self.index.macros().is_incomplete()));
bytes.push(0);
bytes.push(b'|');
bytes.extend_from_slice(
normalize_path(path.parent().unwrap_or(Path::new(".")), cfg!(windows)).as_bytes(),
);
fnv1a64(&bytes)
}
}
fn bodies_of<'pass>(
environments: &'pass [(
&'pass std::sync::Arc<crate::TranslationUnit>,
crate::UnitDefinitions<'pass>,
)],
path: &Path,
) -> Option<(
crate::MacroView<'pass>,
&'pass crate::UnitDefinitions<'pass>,
)> {
environments.iter().find_map(|(unit, definitions)| {
let view = unit.environment_of(path)?;
Some((view, definitions))
})
}
fn mentions_one_of(text: &str, names: &std::collections::HashSet<&str>) -> bool {
words_of(text).any(|word| names.contains(word))
}
fn words_of(text: &str) -> impl Iterator<Item = &str> {
text.split(|character: char| !(character.is_alphanumeric() || character == '_'))
.filter(|word| !word.is_empty())
}
#[cfg(test)]
mod the_plain_shape_tests {
use super::{APlainShape, the_plain_shape};
#[test]
fn a_body_is_settled_without_an_environment_unless_it_begins_with_a_name() {
for (body, expected) in [
("namespace std {", APlainShape::Used),
("}", APlainShape::Used),
("::std::", APlainShape::Used),
("try {", APlainShape::Used),
("((a) > (b) ? (a) : (b))", APlainShape::Settled),
("42", APlainShape::Settled),
("\"text\"", APlainShape::Settled),
("", APlainShape::Settled),
(
"_EXTERN_CXX_WORKAROUND namespace std {",
APlainShape::Unplaceable,
),
("_STD_BEGIN", APlainShape::Unplaceable),
] {
assert_eq!(the_plain_shape(body), expected, "{body:?}");
}
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum APlainShape {
Used,
Settled,
Unplaceable,
}
fn store_it(
database: &Option<std::sync::Arc<crate::cache::cache_db::SummaryCache>>,
path: &Path,
summary: &crate::FileSummary,
generation: u64,
) {
let Some(database) = database else {
return;
};
database.put(summary.key, summary);
database.note_file(
path,
summary.key.content_hash,
summary.key.context_hash,
0,
generation,
);
}
fn the_plain_shape(body: &str) -> APlainShape {
if cpp_parser::shape_of_a_body(body).a_reading_uses_this() {
return APlainShape::Used;
}
let mut characters = body
.trim_start_matches(['\\', ' ', '\t', '\n', '\r'])
.chars();
match characters.next() {
Some(first) if first.is_alphabetic() || first == '_' => APlainShape::Unplaceable,
_ => APlainShape::Settled,
}
}
fn words_mentioned_by(
sources: &std::collections::HashMap<String, String>,
files: &[PathBuf],
) -> std::collections::HashSet<String> {
let mut mentioned: std::collections::HashSet<String> = std::collections::HashSet::new();
for path in files {
let Some(text) = sources.get(&normalize_path(path, cfg!(windows))) else {
continue;
};
for word in words_of(text) {
if !mentioned.contains(word) {
mentioned.insert(word.to_string());
}
}
}
mentioned
}
fn has_unresolved_includes(summary: &FileSummary) -> bool {
summary.includes.iter().any(|include| {
include.resolved.is_none()
&& matches!(include.guard, crate::summary::FactGuard::Unconditional)
})
}
#[cfg(test)]
mod tests;
fn environment_hash(facts: &dyn cpp_parser::MacroFacts) -> u64 {
let mut pairs: Vec<(String, String)> = Vec::new();
facts.for_each_body_in_force(&mut |body| {
pairs.push((body.name.to_string(), body.body.to_string()));
});
pairs.sort();
let mut bytes: Vec<u8> = Vec::new();
for (name, body) in &pairs {
bytes.extend_from_slice(name.as_bytes());
bytes.push(0);
bytes.extend_from_slice(body.as_bytes());
bytes.push(0);
}
crate::cache::fnv1a64(&bytes)
}
fn a_facts_visibility(
guards: &crate::summary::SummaryGuards,
fact: &crate::summary::MacroFact,
table: &crate::preprocess::macros::MacroTable,
answers: &mut std::collections::HashMap<(u32, usize), Option<bool>>,
) -> crate::Visibility {
let crate::summary::FactGuard::Region(innermost) = fact.guard else {
return crate::Visibility::Active;
};
let mut unknown = false;
for at in guards.conditions_of(innermost) {
let Some(region) = guards.region_at(at.region, fact.range.start_offset) else {
unknown = true;
continue;
};
let key = (at.region, region.active_branch);
let answer = match answers.get(&key) {
Some(answer) => *answer,
None => {
let answer = region.visibility(table);
answers.insert(key, answer);
answer
}
};
match answer {
Some(true) => {}
Some(false) => return crate::Visibility::Inactive,
None => unknown = true,
}
}
if unknown {
crate::Visibility::Unknown
} else {
crate::Visibility::Active
}
}