use std::collections::hash_map::Entry;
use std::sync::Arc;
use arcstr::ArcStr;
use futures::future::join_all;
use itertools::Itertools;
use oxc::semantic::{ScopeId, Scoping};
use oxc::transformer_plugins::ReplaceGlobalDefinesConfig;
use oxc_allocator::Address;
use oxc_index::IndexVec;
use rolldown_common::dynamic_import_usage::DynamicImportExportsUsage;
use rolldown_common::{
EcmaRelated, EntryPoint, EntryPointKind, ExternalModule, ExternalModuleTaskResult, FlatOptions,
HybridIndexVec, ImportKind, ImportRecordIdx, ImportRecordMeta, ImporterRecord, Module, ModuleId,
ModuleIdx, ModuleLoaderMsg, ModuleType, NormalModuleTaskResult, PreserveEntrySignatures,
RUNTIME_MODULE_ID, ResolvedId, RuntimeModuleBrief, RuntimeModuleTaskResult, ScanMode,
SourceMapGenMsg, StmtInfoIdx, SymbolRefDb, SymbolRefDbForModule,
};
use rolldown_ecmascript::EcmaAst;
use rolldown_error::{BuildDiagnostic, BuildResult, DiagnosableResolveError};
use rolldown_fs::OsFileSystem;
use rolldown_plugin::SharedPluginDriver;
use rolldown_utils::indexmap::FxIndexSet;
use rolldown_utils::rayon::{IntoParallelIterator, ParallelIterator};
use rolldown_utils::rustc_hash::FxHashSetExt;
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::Instrument;
use crate::types::scan_stage_cache::ScanStageCache;
use crate::utils::load_entry_module::load_entry_module;
use crate::{SharedOptions, SharedResolver};
use super::external_module_task::ExternalModuleTask;
use super::module_task::{ModuleTask, ModuleTaskOwnerRef};
use super::runtime_module_task::RuntimeModuleTask;
use super::task_context::{TaskContext, TaskContextMeta};
pub struct IntermediateNormalModules {
pub modules: HybridIndexVec<ModuleIdx, Option<Module>>,
pub importers: IndexVec<ModuleIdx, Vec<ImporterRecord>>,
pub index_ecma_ast: HybridIndexVec<ModuleIdx, Option<EcmaAst>>,
}
impl IntermediateNormalModules {
pub fn new(is_full_scan: bool, importers: IndexVec<ModuleIdx, Vec<ImporterRecord>>) -> Self {
Self {
modules: if is_full_scan {
HybridIndexVec::IndexVec(IndexVec::default())
} else {
HybridIndexVec::Map(FxHashMap::default())
},
importers,
index_ecma_ast: if is_full_scan {
HybridIndexVec::IndexVec(IndexVec::default())
} else {
HybridIndexVec::Map(FxHashMap::default())
},
}
}
pub fn alloc_ecma_module_idx(&mut self) -> ModuleIdx {
let id = self.modules.push(None);
self.index_ecma_ast.push(None);
self.importers.push(Vec::new());
id
}
pub fn alloc_ecma_module_idx_sparse(&mut self, i: ModuleIdx) -> ModuleIdx {
self.modules.insert(i, None);
self.index_ecma_ast.insert(i, None);
if i >= self.importers.len() {
self.importers.push(Vec::new());
}
i
}
pub fn reset_ecma_module_idx(&mut self) {
self.modules.clear();
self.index_ecma_ast.clear();
}
}
#[derive(Debug, Clone, Copy)]
pub enum VisitState {
Seen(ModuleIdx),
Invalidate(ModuleIdx),
}
impl VisitState {
pub fn idx(self) -> ModuleIdx {
match self {
VisitState::Seen(idx) | VisitState::Invalidate(idx) => idx,
}
}
}
pub struct ModuleLoader<'a> {
pub shared_context: Arc<TaskContext>,
rx: tokio::sync::mpsc::Receiver<ModuleLoaderMsg>,
runtime_idx: ModuleIdx,
remaining: u32,
intermediate_normal_modules: IntermediateNormalModules,
symbol_ref_db: SymbolRefDb,
is_full_scan: bool,
new_added_modules_from_partial_scan: FxIndexSet<ModuleIdx>,
cache: &'a mut ScanStageCache,
pub flat_options: FlatOptions,
pub magic_string_tx: Option<Arc<std::sync::mpsc::Sender<SourceMapGenMsg>>>,
}
pub struct ModuleLoaderOutput {
pub module_table: HybridIndexVec<ModuleIdx, Module>,
pub index_ecma_ast: HybridIndexVec<ModuleIdx, Option<EcmaAst>>,
pub symbol_ref_db: SymbolRefDb,
pub entry_points: Vec<EntryPoint>,
pub runtime: RuntimeModuleBrief,
pub warnings: Vec<BuildDiagnostic>,
pub dynamic_import_exports_usage_map: FxHashMap<ModuleIdx, DynamicImportExportsUsage>,
pub new_added_modules_from_partial_scan: FxIndexSet<ModuleIdx>,
pub overrode_preserve_entry_signature_map: FxHashMap<ModuleIdx, PreserveEntrySignatures>,
pub entry_point_to_reference_ids: FxHashMap<EntryPoint, Vec<ArcStr>>,
pub flat_options: FlatOptions,
}
impl Drop for ModuleLoader<'_> {
fn drop(&mut self) {
self.cache.importers = std::mem::take(&mut self.intermediate_normal_modules.importers);
}
}
impl<'a> ModuleLoader<'a> {
pub fn new(
fs: OsFileSystem,
options: SharedOptions,
resolver: SharedResolver,
plugin_driver: SharedPluginDriver,
cache: &'a mut ScanStageCache,
is_full_scan: bool,
magic_string_tx: Option<Arc<std::sync::mpsc::Sender<SourceMapGenMsg>>>,
) -> BuildResult<Self> {
if is_full_scan {
std::mem::take(cache);
}
let flat_options = FlatOptions::from_shared_options(&options);
let symbol_ref_db = SymbolRefDb::new(options.transform_options.is_jsx_preserve());
let meta = TaskContextMeta {
replace_global_define_config: if options.define.is_empty() {
None
} else {
ReplaceGlobalDefinesConfig::new(&options.define).map(Some).map_err(|errs| {
errs
.into_iter()
.map(|err| BuildDiagnostic::invalid_define_config(err.message.to_string()))
.collect::<Vec<BuildDiagnostic>>()
})?
},
};
let (tx, rx) = tokio::sync::mpsc::channel(1024);
let shared_context = Arc::new(TaskContext { fs, options, resolver, plugin_driver, tx, meta });
let importers = std::mem::take(&mut cache.importers);
let mut intermediate_normal_modules = IntermediateNormalModules::new(is_full_scan, importers);
let runtime_idx = intermediate_normal_modules.alloc_ecma_module_idx();
let remaining = if let Entry::Vacant(e) = cache.module_id_to_idx.entry(RUNTIME_MODULE_ID) {
let task = RuntimeModuleTask::new(runtime_idx, Arc::clone(&shared_context), flat_options);
tokio::spawn(task.run());
e.insert(VisitState::Seen(runtime_idx));
1
} else {
intermediate_normal_modules.reset_ecma_module_idx();
0
};
Ok(Self {
rx,
cache,
remaining,
runtime_idx,
is_full_scan,
shared_context,
symbol_ref_db,
intermediate_normal_modules,
new_added_modules_from_partial_scan: FxIndexSet::default(),
flat_options,
magic_string_tx,
})
}
#[expect(clippy::rc_buffer)]
fn try_spawn_new_task(
&mut self,
resolved_id: ResolvedId,
owner: Option<ModuleTaskOwnerRef<'_>>,
is_user_defined_entry: bool,
assert_module_type: Option<&ModuleType>,
user_defined_entries: &Arc<Vec<(Option<ArcStr>, ResolvedId)>>,
) -> ModuleIdx {
let idx = match self.cache.module_id_to_idx.get(&resolved_id.id) {
Some(VisitState::Seen(idx)) => return *idx,
Some(VisitState::Invalidate(idx)) => {
let idx = *idx;
self.intermediate_normal_modules.alloc_ecma_module_idx_sparse(idx);
self.cache.module_id_to_idx.insert(resolved_id.id.clone(), VisitState::Seen(idx));
idx
}
None if !self.is_full_scan => {
let len = self.cache.module_id_to_idx.len();
let idx = self.intermediate_normal_modules.alloc_ecma_module_idx_sparse(len.into());
self.new_added_modules_from_partial_scan.insert(idx);
self.cache.module_id_to_idx.insert(resolved_id.id.clone(), VisitState::Seen(idx));
idx
}
None => {
let idx = self.intermediate_normal_modules.alloc_ecma_module_idx();
self.cache.module_id_to_idx.insert(resolved_id.id.clone(), VisitState::Seen(idx));
idx
}
};
let ctx = Arc::clone(&self.shared_context);
if resolved_id.external.is_external() {
let task = ExternalModuleTask::new(ctx, idx, resolved_id, Arc::clone(user_defined_entries));
tokio::spawn(task.run().instrument(tracing::info_span!("external_module_task")));
} else {
let task = ModuleTask::new(
ctx,
idx,
resolved_id,
owner.map(Into::into),
is_user_defined_entry,
assert_module_type.cloned(),
self.flat_options,
self.magic_string_tx.clone(),
);
tokio::spawn(task.run().instrument(tracing::info_span!("normal_module_task")));
}
self.remaining += 1;
idx
}
#[tracing::instrument(level = "debug", skip_all)]
#[expect(clippy::too_many_lines)]
pub async fn fetch_modules(
&mut self,
fetch_mode: ScanMode<ResolvedId>,
) -> BuildResult<ModuleLoaderOutput> {
let mut errors = vec![];
let mut all_warnings = vec![];
let user_defined_entries = Arc::new(match fetch_mode {
ScanMode::Full => self.resolve_user_defined_entries().await?,
ScanMode::Partial(_) => vec![],
});
let entries_count = user_defined_entries.len() + 1;
self.intermediate_normal_modules.modules.reserve(entries_count);
self.intermediate_normal_modules.index_ecma_ast.reserve(entries_count);
let mut entry_points = FxIndexSet::default();
let mut user_defined_entry_ids = FxHashSet::with_capacity(user_defined_entries.len());
for (name, resolved_id) in user_defined_entries.iter().cloned() {
let idx = self.try_spawn_new_task(resolved_id, None, true, None, &user_defined_entries);
user_defined_entry_ids.insert(idx);
entry_points.insert(EntryPoint {
idx,
name,
kind: EntryPointKind::UserDefined,
file_name: None,
related_stmt_infos: vec![],
});
}
if self.is_full_scan && self.shared_context.options.experimental.is_incremental_build_enabled()
{
self
.cache
.user_defined_entry
.extend(user_defined_entries.iter().map(|(_, resolved_id)| resolved_id.id.clone()));
}
for resolved_id in fetch_mode.iter().cloned() {
self.shared_context.plugin_driver.invalidate_context_load_module(&resolved_id.id);
if let Entry::Occupied(mut occ) = self.cache.module_id_to_idx.entry(resolved_id.id.clone()) {
let idx = occ.get().idx();
occ.insert(VisitState::Invalidate(idx));
}
let is_user_defined_entry = self.cache.user_defined_entry.contains(&resolved_id.id);
self.try_spawn_new_task(
resolved_id,
None,
is_user_defined_entry,
None,
&user_defined_entries,
);
}
let mut dynamic_import_entry_ids: FxHashMap<
ModuleIdx,
Vec<(ModuleIdx, StmtInfoIdx, Address, ImportRecordIdx)>,
> = FxHashMap::default();
let mut dynamic_import_exports_usage_pairs = vec![];
let mut extra_entry_points = vec![];
let mut entry_point_to_reference_ids: FxHashMap<EntryPoint, Vec<ArcStr>> = FxHashMap::default();
let mut runtime_brief = None;
let mut overrode_preserve_entry_signature_map = FxHashMap::default();
while self.remaining > 0 {
let Some(msg) = self.rx.recv().await else {
break;
};
match msg {
ModuleLoaderMsg::NormalModuleDone(task_result) => {
let NormalModuleTaskResult {
mut module,
ecma_related: EcmaRelated { ast, symbols, mut dynamic_import_rec_exports_usage },
resolved_deps,
raw_import_records,
warnings,
} = *task_result;
all_warnings.extend(warnings);
let module_idx = module.idx();
if user_defined_entry_ids.contains(&module_idx) {
let normal_module = module.as_normal_mut().expect("should be normal module");
normal_module.is_user_defined_entry = true;
}
if !self.is_full_scan
&& let Some(previous_module) =
self.cache.get_snapshot().module_table.modules.get(module_idx)
{
for rec in previous_module.import_records() {
self.intermediate_normal_modules.importers[rec.resolved_module]
.retain(|v| v.importer_idx != module_idx);
}
}
let normal_module = module.as_normal().unwrap();
let mut import_records = IndexVec::with_capacity(raw_import_records.len());
for ((rec_idx, mut raw_rec), resolved_id) in
raw_import_records.into_iter_enumerated().zip(resolved_deps)
{
if self.shared_context.options.experimental.vite_mode.unwrap_or_default()
&& resolved_id.id.as_str().ends_with(".json")
{
raw_rec.meta.insert(ImportRecordMeta::JsonModule);
}
let idx = self.try_spawn_new_task(
resolved_id,
Some(ModuleTaskOwnerRef::new(normal_module, raw_rec.span)),
false,
raw_rec.asserted_module_type.as_ref(),
&user_defined_entries,
);
self.intermediate_normal_modules.importers[idx].push(ImporterRecord {
kind: raw_rec.kind,
importer_path: module.id().clone(),
importer_idx: module_idx,
});
if let Some(usage) = dynamic_import_rec_exports_usage.remove(&rec_idx) {
dynamic_import_exports_usage_pairs.push((idx, usage));
}
if matches!(raw_rec.kind, ImportKind::DynamicImport)
&& !user_defined_entry_ids.contains(&idx)
{
match dynamic_import_entry_ids.entry(idx) {
Entry::Vacant(vac) => match raw_rec.dynamic_import_expr_info.as_ref() {
Some(info) => {
vac.insert(vec![(module_idx, info.stmt_info_idx, info.address, rec_idx)]);
}
None => {
vac.insert(vec![]);
}
},
Entry::Occupied(mut occ) => {
if let Some(info) = raw_rec.dynamic_import_expr_info.as_ref() {
occ.get_mut().push((module_idx, info.stmt_info_idx, info.address, rec_idx));
}
}
}
}
import_records.push(raw_rec.into_resolved(idx));
}
module.set_import_records(import_records);
*self.intermediate_normal_modules.index_ecma_ast.get_mut(module_idx) = Some(ast);
*self.intermediate_normal_modules.modules.get_mut(module_idx) = Some(module);
self.symbol_ref_db.store_local_db(module_idx, symbols);
self.remaining -= 1;
}
ModuleLoaderMsg::ExternalModuleDone(task_result) => {
let ExternalModuleTaskResult {
id,
name,
idx,
identifier_name,
side_effects,
need_renormalize_render_path,
} = *task_result;
self.symbol_ref_db.store_local_db(
idx,
SymbolRefDbForModule::new(Scoping::default(), idx, ScopeId::new(0)),
);
let symbol_ref = self.symbol_ref_db.create_facade_root_symbol_ref(idx, &identifier_name);
let external_module = Module::External(Box::new(ExternalModule::new(
idx,
id,
name,
identifier_name,
side_effects,
symbol_ref,
need_renormalize_render_path,
)));
*self.intermediate_normal_modules.modules.get_mut(idx) = Some(external_module);
self.remaining -= 1;
}
ModuleLoaderMsg::RuntimeNormalModuleDone(task_result) => {
let RuntimeModuleTaskResult {
local_symbol_ref_db,
mut module,
runtime,
ast,
raw_import_records,
resolved_deps,
} = *task_result;
let mut import_records = IndexVec::with_capacity(raw_import_records.len());
for (raw_rec, info) in raw_import_records.into_iter().zip(resolved_deps) {
let id = self.try_spawn_new_task(
info,
None,
false,
raw_rec.asserted_module_type.as_ref(),
&user_defined_entries,
);
self.intermediate_normal_modules.importers[id].push(ImporterRecord {
kind: raw_rec.kind,
importer_path: module.id.clone(),
importer_idx: module.idx,
});
import_records.push(raw_rec.into_resolved(id));
}
module.import_records = import_records;
*self.intermediate_normal_modules.modules.get_mut(self.runtime_idx) = Some(module.into());
*self.intermediate_normal_modules.index_ecma_ast.get_mut(self.runtime_idx) = Some(ast);
self.symbol_ref_db.store_local_db(self.runtime_idx, local_symbol_ref_db);
self.remaining -= 1;
runtime_brief = Some(runtime);
}
ModuleLoaderMsg::FetchModule(resolve_id) => {
self.try_spawn_new_task(*resolve_id, None, false, None, &user_defined_entries);
}
ModuleLoaderMsg::AddEntryModule(msg) => {
let data = msg.chunk;
let result = load_entry_module(
&self.shared_context.resolver,
&self.shared_context.plugin_driver,
&data.id,
data.importer.as_deref(),
)
.await;
let module_idx = match result {
Ok(resolved_id) => {
let idx =
self.try_spawn_new_task(resolved_id, None, true, None, &user_defined_entries);
if let Some(module) = self
.intermediate_normal_modules
.modules
.get_mut(idx)
.as_mut()
.and_then(|module| module.as_normal_mut())
{
module.is_user_defined_entry = true;
}
idx
}
Err(e) => {
errors.push(e);
continue;
}
};
if let Some(preserve_entry_signatures) = data.preserve_entry_signatures {
overrode_preserve_entry_signature_map.insert(module_idx, preserve_entry_signatures);
}
user_defined_entry_ids.insert(module_idx);
let entry = EntryPoint {
name: data.name.clone(),
idx: module_idx,
kind: EntryPointKind::EmittedUserDefined,
file_name: data.file_name.clone(),
related_stmt_infos: vec![],
};
entry_point_to_reference_ids
.entry(entry.clone())
.or_default()
.push(msg.reference_id.clone());
extra_entry_points.push(entry);
}
ModuleLoaderMsg::BuildErrors(e) => {
errors.extend(e);
self.remaining -= 1;
}
}
}
if !errors.is_empty() {
for error in &mut errors {
if let Some(resolve_error) = error.downcast_mut::<DiagnosableResolveError>() {
let chain = self
.trace_import_chain_from_modules(&resolve_error.importer_id, &user_defined_entry_ids);
if !chain.is_empty() {
resolve_error.import_chain = Some(chain);
}
}
}
return Err(errors.into());
}
if let Some(tx) = self.magic_string_tx.as_ref() {
tx.send(SourceMapGenMsg::Terminate).expect(
"SourceMapGen: failed to send Terminate message - sourcemap worker thread died unexpectedly"
);
}
let dynamic_import_exports_usage_map = dynamic_import_exports_usage_pairs.into_iter().fold(
FxHashMap::default(),
|mut acc, (idx, usage)| {
match acc.entry(idx) {
Entry::Vacant(vac) => {
vac.insert(usage);
}
Entry::Occupied(mut occ) => {
occ.get_mut().merge(usage);
}
}
acc
},
);
let mut idx_of_module_info_need_update = vec![];
let is_dense_index_vec = self.intermediate_normal_modules.modules.is_index_vec();
let modules_iter = std::mem::take(&mut self.intermediate_normal_modules.modules)
.into_iter_enumerated()
.into_iter()
.map(|(idx, module)| {
let mut module = module.expect("Module tasks did't complete as expected");
if let Some(module) = module.as_normal_mut() {
let importers = &self.intermediate_normal_modules.importers[idx];
for importer in importers {
if importer.kind.is_static() {
module.importers.insert(importer.importer_path.clone());
module.importers_idx.insert(importer.importer_idx);
} else {
module.dynamic_importers.insert(importer.importer_path.clone());
}
}
if !importers.is_empty() {
idx_of_module_info_need_update.push(idx);
}
}
(idx, module)
});
let module_table = if is_dense_index_vec {
let vec = modules_iter.map(|(_, module)| module).collect();
HybridIndexVec::IndexVec(IndexVec::from_vec(vec))
} else {
let map = modules_iter.collect::<FxHashMap<_, _>>();
HybridIndexVec::Map(map)
};
idx_of_module_info_need_update.extend(extra_entry_points.iter().map(|item| item.idx));
idx_of_module_info_need_update.into_par_iter().for_each(|idx| {
let module = module_table.get(idx);
let Some(module) = module.as_normal() else {
return;
};
self
.shared_context
.plugin_driver
.set_module_info(&module.id, Arc::new(module.to_module_info(None)));
});
let emitted_entry_indices: FxHashSet<ModuleIdx> =
extra_entry_points.iter().map(|e| e.idx).collect();
for (idx, related_stmt_infos) in dynamic_import_entry_ids {
if !emitted_entry_indices.contains(&idx) {
entry_points.insert(EntryPoint {
name: None,
idx,
kind: EntryPointKind::DynamicImport,
file_name: None,
related_stmt_infos,
});
}
}
entry_points.extend(extra_entry_points);
if entry_points.is_empty() && self.is_full_scan {
Err(BuildDiagnostic::invalid_option(rolldown_error::InvalidOptionType::NoEntryPoint))?;
}
let entry_points = entry_points.into_iter().collect_vec();
let runtime = if self.is_full_scan {
tracing::debug!("changed_resolved_ids: {fetch_mode:#?}");
runtime_brief.expect("Failed to find runtime module. This should not happen")
} else {
RuntimeModuleBrief::dummy()
};
Ok(ModuleLoaderOutput {
runtime,
entry_points,
module_table,
warnings: all_warnings,
dynamic_import_exports_usage_map,
overrode_preserve_entry_signature_map,
entry_point_to_reference_ids,
symbol_ref_db: std::mem::take(&mut self.symbol_ref_db),
index_ecma_ast: std::mem::take(&mut self.intermediate_normal_modules.index_ecma_ast),
new_added_modules_from_partial_scan: std::mem::take(
&mut self.new_added_modules_from_partial_scan,
),
flat_options: self.flat_options,
})
}
#[tracing::instrument(target = "devtool", level = "debug", skip_all)]
pub async fn resolve_user_defined_entries(
&self,
) -> BuildResult<Vec<(Option<ArcStr>, ResolvedId)>> {
let resolved_ids =
join_all(self.shared_context.options.input.iter().map(|input_item| async move {
let resolved = load_entry_module(
&self.shared_context.resolver,
&self.shared_context.plugin_driver,
&input_item.import,
None,
)
.await;
resolved.map(|info| (input_item.name.as_ref().map(Into::into), info))
}))
.await;
let mut ret = Vec::with_capacity(self.shared_context.options.input.len());
let mut errors = vec![];
for resolve_id in resolved_ids {
match resolve_id {
Ok(item) => {
ret.push(item);
}
Err(e) => errors.push(e),
}
}
if !errors.is_empty() {
Err(errors)?;
}
Ok(ret)
}
fn trace_import_chain_from_modules(
&self,
importer_id: &str,
user_defined_entry_ids: &FxHashSet<ModuleIdx>,
) -> Vec<String> {
let importer_module_id = ModuleId::new(importer_id);
let Some(visit_state) = self.cache.module_id_to_idx.get(&importer_module_id) else {
return vec![];
};
let start_idx = visit_state.idx();
let mut chain = Vec::new();
let mut visited = FxHashSet::default();
let mut current = Some(start_idx);
while let Some(idx) = current {
if visited.contains(&idx) {
break;
}
visited.insert(idx);
let module_opt = self.intermediate_normal_modules.modules.get(idx);
if let Some(module) = module_opt {
if let Some(normal) = module.as_normal() {
chain.push(normal.id.to_string());
}
}
if user_defined_entry_ids.contains(&idx) {
break;
}
current = self
.intermediate_normal_modules
.importers
.get(idx)
.and_then(|importers| importers.first().map(|rec| rec.importer_idx));
}
chain
}
}