use crate::{
artifact_output::Artifacts,
artifacts::{Settings, VersionedFilteredSources, VersionedSources},
buildinfo::RawBuildInfo,
cache::ArtifactsCache,
error::Result,
filter::SparseOutputFilter,
output::AggregatedCompilerOutput,
report,
resolver::GraphEdges,
ArtifactOutput, CompilerInput, Graph, Project, ProjectCompileOutput, ProjectPathsConfig, Solc,
Sources,
};
use rayon::prelude::*;
use std::{collections::btree_map::BTreeMap, path::PathBuf, time::Instant};
use tracing::trace;
#[derive(Debug)]
pub struct ProjectCompiler<'a, T: ArtifactOutput> {
edges: GraphEdges,
project: &'a Project<T>,
sources: CompilerSources,
sparse_output: SparseOutputFilter,
}
impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
#[cfg(all(feature = "svm-solc", not(target_arch = "wasm32")))]
pub fn new(project: &'a Project<T>) -> Result<Self> {
Self::with_sources(project, project.paths.read_input_files()?)
}
#[cfg(all(feature = "svm-solc", not(target_arch = "wasm32")))]
pub fn with_sources(project: &'a Project<T>, sources: Sources) -> Result<Self> {
let graph = Graph::resolve_sources(&project.paths, sources)?;
let (versions, edges) = graph.into_sources_by_version(project.offline)?;
let sources_by_version = versions.get(project)?;
let sources = if project.solc_jobs > 1 && sources_by_version.len() > 1 {
CompilerSources::Parallel(sources_by_version, project.solc_jobs)
} else {
CompilerSources::Sequential(sources_by_version)
};
Ok(Self { edges, project, sources, sparse_output: Default::default() })
}
pub fn with_sources_and_solc(
project: &'a Project<T>,
sources: Sources,
solc: Solc,
) -> Result<Self> {
let version = solc.version()?;
let (sources, edges) = Graph::resolve_sources(&project.paths, sources)?.into_sources();
let solc = project.configure_solc_with_version(
solc,
Some(version.clone()),
edges.include_paths().clone(),
);
let sources_by_version = BTreeMap::from([(solc, (version, sources))]);
let sources = CompilerSources::Sequential(sources_by_version);
Ok(Self { edges, project, sources, sparse_output: Default::default() })
}
pub fn with_sparse_output(mut self, sparse_output: impl Into<SparseOutputFilter>) -> Self {
self.sparse_output = sparse_output.into();
self
}
pub fn compile(self) -> Result<ProjectCompileOutput<T>> {
let slash_paths = self.project.slash_paths;
let mut output = self.preprocess()?.compile()?.write_artifacts()?.write_cache()?;
if slash_paths {
output.slash_paths();
}
Ok(output)
}
fn preprocess(self) -> Result<PreprocessedState<'a, T>> {
trace!("preprocessing");
let Self { edges, project, mut sources, sparse_output } = self;
sources.slash_paths();
let mut cache = ArtifactsCache::new(project, edges)?;
let sources = sources.filtered(&mut cache);
Ok(PreprocessedState { sources, cache, sparse_output })
}
}
#[derive(Debug)]
struct PreprocessedState<'a, T: ArtifactOutput> {
sources: FilteredCompilerSources,
cache: ArtifactsCache<'a, T>,
sparse_output: SparseOutputFilter,
}
impl<'a, T: ArtifactOutput> PreprocessedState<'a, T> {
fn compile(self) -> Result<CompiledState<'a, T>> {
trace!("compiling");
let PreprocessedState { sources, cache, sparse_output } = self;
let project = cache.project();
let mut output = sources.compile(
&project.solc_config.settings,
&project.paths,
sparse_output,
cache.graph(),
project.build_info,
)?;
output.join_all(cache.project().root());
Ok(CompiledState { output, cache })
}
}
#[derive(Debug)]
struct CompiledState<'a, T: ArtifactOutput> {
output: AggregatedCompilerOutput,
cache: ArtifactsCache<'a, T>,
}
impl<'a, T: ArtifactOutput> CompiledState<'a, T> {
#[tracing::instrument(skip_all, name = "write-artifacts")]
fn write_artifacts(self) -> Result<ArtifactsState<'a, T>> {
let CompiledState { output, cache } = self;
let project = cache.project();
let ctx = cache.output_ctx();
let compiled_artifacts = if project.no_artifacts {
project.artifacts_handler().output_to_artifacts(
&output.contracts,
&output.sources,
ctx,
&project.paths,
)
} else if output.has_error(&project.ignored_error_codes, &project.compiler_severity_filter)
{
trace!("skip writing cache file due to solc errors: {:?}", output.errors);
project.artifacts_handler().output_to_artifacts(
&output.contracts,
&output.sources,
ctx,
&project.paths,
)
} else {
trace!(
"handling artifact output for {} contracts and {} sources",
output.contracts.len(),
output.sources.len()
);
let artifacts = project.artifacts_handler().on_output(
&output.contracts,
&output.sources,
&project.paths,
ctx,
)?;
output.write_build_infos(project.build_info_path())?;
artifacts
};
Ok(ArtifactsState { output, cache, compiled_artifacts })
}
}
#[derive(Debug)]
struct ArtifactsState<'a, T: ArtifactOutput> {
output: AggregatedCompilerOutput,
cache: ArtifactsCache<'a, T>,
compiled_artifacts: Artifacts<T::Artifact>,
}
impl<'a, T: ArtifactOutput> ArtifactsState<'a, T> {
fn write_cache(self) -> Result<ProjectCompileOutput<T>> {
let ArtifactsState { output, cache, compiled_artifacts } = self;
let project = cache.project();
let ignored_error_codes = project.ignored_error_codes.clone();
let compiler_severity_filter = project.compiler_severity_filter;
let has_error = output.has_error(&ignored_error_codes, &compiler_severity_filter);
let skip_write_to_disk = project.no_artifacts || has_error;
trace!(has_error, project.no_artifacts, skip_write_to_disk, cache_path=?project.cache_path(),"prepare writing cache file");
let cached_artifacts = cache.consume(&compiled_artifacts, !skip_write_to_disk)?;
Ok(ProjectCompileOutput {
compiler_output: output,
compiled_artifacts,
cached_artifacts,
ignored_error_codes,
compiler_severity_filter,
})
}
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
enum CompilerSources {
Sequential(VersionedSources),
Parallel(VersionedSources, usize),
}
impl CompilerSources {
fn slash_paths(&mut self) {
#[cfg(windows)]
{
use path_slash::PathBufExt;
fn slash_versioned_sources(v: &mut VersionedSources) {
for (_, (_, sources)) in v {
*sources = std::mem::take(sources)
.into_iter()
.map(|(path, source)| {
(PathBuf::from(path.to_slash_lossy().as_ref()), source)
})
.collect()
}
}
match self {
CompilerSources::Sequential(v) => slash_versioned_sources(v),
CompilerSources::Parallel(v, _) => slash_versioned_sources(v),
};
}
}
fn filtered<T: ArtifactOutput>(self, cache: &mut ArtifactsCache<T>) -> FilteredCompilerSources {
fn filtered_sources<T: ArtifactOutput>(
sources: VersionedSources,
cache: &mut ArtifactsCache<T>,
) -> VersionedFilteredSources {
sources.iter().for_each(|(_, (_, sources))| {
cache.fill_content_hashes(sources);
});
sources
.into_iter()
.map(|(solc, (version, sources))| {
trace!("Filtering {} sources for {}", sources.len(), version);
let sources = cache.filter(sources, &version);
trace!(
"Detected {} dirty sources {:?}",
sources.dirty().count(),
sources.dirty_files().collect::<Vec<_>>()
);
(solc, (version, sources))
})
.collect()
}
match self {
CompilerSources::Sequential(s) => {
FilteredCompilerSources::Sequential(filtered_sources(s, cache))
}
CompilerSources::Parallel(s, j) => {
FilteredCompilerSources::Parallel(filtered_sources(s, cache), j)
}
}
}
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
enum FilteredCompilerSources {
Sequential(VersionedFilteredSources),
Parallel(VersionedFilteredSources, usize),
}
impl FilteredCompilerSources {
fn compile(
self,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFilter,
graph: &GraphEdges,
create_build_info: bool,
) -> Result<AggregatedCompilerOutput> {
match self {
FilteredCompilerSources::Sequential(input) => {
compile_sequential(input, settings, paths, sparse_output, graph, create_build_info)
}
FilteredCompilerSources::Parallel(input, j) => {
compile_parallel(input, j, settings, paths, sparse_output, graph, create_build_info)
}
}
}
#[cfg(test)]
#[allow(unused)]
fn sources(&self) -> &VersionedFilteredSources {
match self {
FilteredCompilerSources::Sequential(v) => v,
FilteredCompilerSources::Parallel(v, _) => v,
}
}
}
fn compile_sequential(
input: VersionedFilteredSources,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFilter,
graph: &GraphEdges,
create_build_info: bool,
) -> Result<AggregatedCompilerOutput> {
let mut aggregated = AggregatedCompilerOutput::default();
trace!("compiling {} jobs sequentially", input.len());
for (solc, (version, filtered_sources)) in input {
if filtered_sources.is_empty() {
trace!("skip solc {} {} for empty sources set", solc.as_ref().display(), version);
continue
}
trace!(
"compiling {} sources with solc \"{}\" {:?}",
filtered_sources.len(),
solc.as_ref().display(),
solc.args
);
let dirty_files: Vec<PathBuf> = filtered_sources.dirty_files().cloned().collect();
let mut opt_settings = settings.clone();
let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings, graph);
for input in CompilerInput::with_sources(sources) {
let actually_dirty = input
.sources
.keys()
.filter(|f| dirty_files.contains(f))
.cloned()
.collect::<Vec<_>>();
if actually_dirty.is_empty() {
trace!(
"skip solc {} {} compilation of {} compiler input due to empty source set",
solc.as_ref().display(),
version,
input.language
);
continue
}
let input = input
.settings(opt_settings.clone())
.normalize_evm_version(&version)
.with_remappings(paths.remappings.clone())
.with_base_path(&paths.root)
.sanitized(&version);
trace!(
"calling solc `{}` with {} sources {:?}",
version,
input.sources.len(),
input.sources.keys()
);
let start = Instant::now();
report::solc_spawn(&solc, &version, &input, &actually_dirty);
let output = solc.compile(&input)?;
report::solc_success(&solc, &version, &output, &start.elapsed());
trace!("compiled input, output has error: {}", output.has_error());
trace!("received compiler output: {:?}", output.contracts.keys());
if create_build_info {
let build_info = RawBuildInfo::new(&input, &output, &version)?;
aggregated.build_infos.insert(version.clone(), build_info);
}
aggregated.extend(version.clone(), output);
}
}
Ok(aggregated)
}
fn compile_parallel(
input: VersionedFilteredSources,
num_jobs: usize,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFilter,
graph: &GraphEdges,
create_build_info: bool,
) -> Result<AggregatedCompilerOutput> {
debug_assert!(num_jobs > 1);
trace!("compile {} sources in parallel using up to {} solc jobs", input.len(), num_jobs);
let mut jobs = Vec::with_capacity(input.len());
for (solc, (version, filtered_sources)) in input {
if filtered_sources.is_empty() {
trace!("skip solc {} {} for empty sources set", solc.as_ref().display(), version);
continue
}
let dirty_files: Vec<PathBuf> = filtered_sources.dirty_files().cloned().collect();
let mut opt_settings = settings.clone();
let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings, graph);
for input in CompilerInput::with_sources(sources) {
let actually_dirty = input
.sources
.keys()
.filter(|f| dirty_files.contains(f))
.cloned()
.collect::<Vec<_>>();
if actually_dirty.is_empty() {
trace!(
"skip solc {} {} compilation of {} compiler input due to empty source set",
solc.as_ref().display(),
version,
input.language
);
continue
}
let job = input
.settings(settings.clone())
.normalize_evm_version(&version)
.with_remappings(paths.remappings.clone())
.with_base_path(&paths.root)
.sanitized(&version);
jobs.push((solc.clone(), version.clone(), job, actually_dirty))
}
}
let scoped_report = report::get_default(|reporter| reporter.clone());
let pool = rayon::ThreadPoolBuilder::new().num_threads(num_jobs).build().unwrap();
let outputs = pool.install(move || {
jobs.into_par_iter()
.map(move |(solc, version, input, actually_dirty)| {
let _guard = report::set_scoped(&scoped_report);
trace!(
"calling solc `{}` {:?} with {} sources: {:?}",
version,
solc.args,
input.sources.len(),
input.sources.keys()
);
let start = Instant::now();
report::solc_spawn(&solc, &version, &input, &actually_dirty);
solc.compile(&input).map(move |output| {
report::solc_success(&solc, &version, &output, &start.elapsed());
(version, input, output)
})
})
.collect::<Result<Vec<_>>>()
})?;
let mut aggregated = AggregatedCompilerOutput::default();
for (version, input, output) in outputs {
if create_build_info {
let build_info = RawBuildInfo::new(&input, &output, &version)?;
aggregated.build_infos.insert(version.clone(), build_info);
}
aggregated.extend(version, output);
}
Ok(aggregated)
}
#[cfg(test)]
#[cfg(all(feature = "project-util", feature = "svm-solc"))]
mod tests {
use super::*;
use crate::{project_util::TempProject, MinimalCombinedArtifacts};
#[allow(unused)]
fn init_tracing() {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.try_init()
.ok();
}
#[test]
fn can_preprocess() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test-data/dapp-sample");
let project =
Project::builder().paths(ProjectPathsConfig::dapptools(root).unwrap()).build().unwrap();
let compiler = ProjectCompiler::new(&project).unwrap();
let prep = compiler.preprocess().unwrap();
let cache = prep.cache.as_cached().unwrap();
assert_eq!(cache.dirty_source_files.len(), 3);
assert!(cache.filtered.is_empty());
assert!(cache.cache.is_empty());
let compiled = prep.compile().unwrap();
assert_eq!(compiled.output.contracts.files().count(), 3);
}
#[test]
fn can_detect_cached_files() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test-data/dapp-sample");
let paths = ProjectPathsConfig::builder().sources(root.join("src")).lib(root.join("lib"));
let project = TempProject::<MinimalCombinedArtifacts>::new(paths).unwrap();
let compiled = project.compile().unwrap();
compiled.assert_success();
let inner = project.project();
let compiler = ProjectCompiler::new(inner).unwrap();
let prep = compiler.preprocess().unwrap();
assert!(prep.cache.as_cached().unwrap().dirty_source_files.is_empty())
}
#[test]
fn can_recompile_with_optimized_output() {
let tmp = TempProject::dapptools().unwrap();
tmp.add_source(
"A",
r#"
pragma solidity ^0.8.10;
import "./B.sol";
contract A {}
"#,
)
.unwrap();
tmp.add_source(
"B",
r#"
pragma solidity ^0.8.10;
contract B {
function hello() public {}
}
import "./C.sol";
"#,
)
.unwrap();
tmp.add_source(
"C",
r"
pragma solidity ^0.8.10;
contract C {
function hello() public {}
}
",
)
.unwrap();
let compiled = tmp.compile().unwrap();
compiled.assert_success();
tmp.artifacts_snapshot().unwrap().assert_artifacts_essentials_present();
tmp.add_source(
"A",
r#"
pragma solidity ^0.8.10;
import "./B.sol";
contract A {
function testExample() public {}
}
"#,
)
.unwrap();
let compiler = ProjectCompiler::new(tmp.project()).unwrap();
let state = compiler.preprocess().unwrap();
let sources = state.sources.sources();
assert_eq!(sources.len(), 1);
let (_, filtered) = sources.values().next().unwrap();
assert_eq!(filtered.0.len(), 3);
assert_eq!(filtered.dirty().count(), 1);
assert!(filtered.dirty_files().next().unwrap().ends_with("A.sol"));
let state = state.compile().unwrap();
assert_eq!(state.output.sources.len(), 3);
for (f, source) in state.output.sources.sources() {
if f.ends_with("A.sol") {
assert!(source.ast.is_some());
} else {
assert!(source.ast.is_none());
}
}
assert_eq!(state.output.contracts.len(), 1);
let (a, c) = state.output.contracts_iter().next().unwrap();
assert_eq!(a, "A");
assert!(c.abi.is_some() && c.evm.is_some());
let state = state.write_artifacts().unwrap();
assert_eq!(state.compiled_artifacts.as_ref().len(), 1);
let out = state.write_cache().unwrap();
let artifacts: Vec<_> = out.into_artifacts().collect();
assert_eq!(artifacts.len(), 3);
for (_, artifact) in artifacts {
let c = artifact.into_contract_bytecode();
assert!(c.abi.is_some() && c.bytecode.is_some() && c.deployed_bytecode.is_some());
}
tmp.artifacts_snapshot().unwrap().assert_artifacts_essentials_present();
}
#[test]
#[ignore]
fn can_compile_real_project() {
init_tracing();
let paths = ProjectPathsConfig::builder()
.root("../../foundry-integration-tests/testdata/solmate")
.build()
.unwrap();
let project = Project::builder().paths(paths).build().unwrap();
let compiler = ProjectCompiler::new(&project).unwrap();
let _out = compiler.compile().unwrap();
}
}