use std::collections::hash_map::Entry;
use std::sync::Arc;
use arcstr::ArcStr;
use itertools::Itertools;
use oxc::semantic::{ScopeId, Scoping};
use oxc::transformer_plugins::ReplaceGlobalDefinesConfig;
use oxc_index::IndexVec;
use rolldown_common::SourceMapGenMsg;
use rolldown_common::dynamic_import_usage::DynamicImportExportsUsage;
use rolldown_common::{
EcmaRelated, EntryPoint, EntryPointKind, ExternalModule, ExternalModuleTaskResult, FlatOptions,
HybridIndexVec, ImportKind, ImportRecordIdx, ImportRecordMeta, ImporterRecord, Module, ModuleId,
ModuleIdx, ModuleLoaderMsg, ModuleType, NormalModuleTaskResult, PreserveEntrySignatures,
RUNTIME_MODULE_KEY, ResolvedId, RuntimeModuleBrief, RuntimeModuleTaskResult, ScanMode,
StmtInfoIdx, SymbolRef, SymbolRefDb, SymbolRefDbForModule,
};
use rolldown_ecmascript::EcmaAst;
use rolldown_error::{BuildDiagnostic, BuildResult};
use rolldown_fs::OsFileSystem;
use rolldown_plugin::SharedPluginDriver;
use rolldown_utils::indexmap::FxIndexSet;
use rolldown_utils::rayon::{IntoParallelIterator, ParallelIterator};
use rolldown_utils::rustc_hash::FxHashSetExt;
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::Instrument;
use crate::module_loader::task_context::TaskContext;
use crate::stages::scan_stage::resolve_user_defined_entries;
use crate::types::scan_stage_cache::ScanStageCache;
use crate::utils::load_entry_module::load_entry_module;
use crate::{SharedOptions, SharedResolver};
use super::external_module_task::ExternalModuleTask;
use super::module_task::{ModuleTask, ModuleTaskOwner};
use super::runtime_module_task::RuntimeModuleTask;
use super::task_context::TaskContextMeta;
pub struct IntermediateNormalModules {
pub modules: HybridIndexVec<ModuleIdx, Option<Module>>,
pub importers: IndexVec<ModuleIdx, Vec<ImporterRecord>>,
pub index_ecma_ast: HybridIndexVec<ModuleIdx, Option<EcmaAst>>,
}
impl IntermediateNormalModules {
pub fn new(is_full_scan: bool, importers: IndexVec<ModuleIdx, Vec<ImporterRecord>>) -> Self {
Self {
modules: if is_full_scan {
HybridIndexVec::IndexVec(IndexVec::default())
} else {
HybridIndexVec::Map(FxHashMap::default())
},
importers,
index_ecma_ast: if is_full_scan {
HybridIndexVec::IndexVec(IndexVec::default())
} else {
HybridIndexVec::Map(FxHashMap::default())
},
}
}
pub fn alloc_ecma_module_idx(&mut self) -> ModuleIdx {
let id = self.modules.push(None);
self.index_ecma_ast.push(None);
self.importers.push(Vec::new());
id
}
pub fn alloc_ecma_module_idx_sparse(&mut self, i: ModuleIdx) -> ModuleIdx {
self.modules.insert(i, None);
self.index_ecma_ast.insert(i, None);
if i >= self.importers.len() {
self.importers.push(Vec::new());
}
i
}
pub fn reset_ecma_module_idx(&mut self) {
self.modules.clear();
self.index_ecma_ast.clear();
}
}
#[derive(Debug, Clone, Copy)]
pub enum VisitState {
Seen(ModuleIdx),
Invalidate(ModuleIdx),
}
impl VisitState {
pub fn idx(self) -> ModuleIdx {
match self {
VisitState::Seen(idx) | VisitState::Invalidate(idx) => idx,
}
}
}
pub struct ModuleLoader<'a> {
options: SharedOptions,
shared_context: Arc<TaskContext>,
pub tx: tokio::sync::mpsc::Sender<ModuleLoaderMsg>,
rx: tokio::sync::mpsc::Receiver<ModuleLoaderMsg>,
runtime_id: ModuleIdx,
remaining: u32,
intermediate_normal_modules: IntermediateNormalModules,
symbol_ref_db: SymbolRefDb,
is_full_scan: bool,
new_added_modules_from_partial_scan: FxIndexSet<ModuleIdx>,
cache: &'a mut ScanStageCache,
pub flat_options: FlatOptions,
pub magic_string_tx: Option<Arc<std::sync::mpsc::Sender<SourceMapGenMsg>>>,
}
pub struct ModuleLoaderOutput {
pub module_table: HybridIndexVec<ModuleIdx, Module>,
pub index_ecma_ast: HybridIndexVec<ModuleIdx, Option<EcmaAst>>,
pub symbol_ref_db: SymbolRefDb,
pub entry_points: Vec<EntryPoint>,
pub runtime: RuntimeModuleBrief,
pub warnings: Vec<BuildDiagnostic>,
pub dynamic_import_exports_usage_map: FxHashMap<ModuleIdx, DynamicImportExportsUsage>,
pub new_added_modules_from_partial_scan: FxIndexSet<ModuleIdx>,
pub safely_merge_cjs_ns_map: FxHashMap<ModuleIdx, Vec<SymbolRef>>,
pub overrode_preserve_entry_signature_map: FxHashMap<ModuleIdx, PreserveEntrySignatures>,
pub entry_point_to_reference_ids: FxHashMap<EntryPoint, Vec<ArcStr>>,
pub flat_options: FlatOptions,
}
impl Drop for ModuleLoader<'_> {
fn drop(&mut self) {
self.cache.importers = std::mem::take(&mut self.intermediate_normal_modules.importers);
}
}
impl<'a> ModuleLoader<'a> {
pub fn new(
fs: OsFileSystem,
options: SharedOptions,
resolver: SharedResolver,
plugin_driver: SharedPluginDriver,
cache: &'a mut ScanStageCache,
is_full_scan: bool,
magic_string_tx: Option<Arc<std::sync::mpsc::Sender<SourceMapGenMsg>>>,
) -> BuildResult<Self> {
if is_full_scan {
std::mem::take(cache);
}
let flat_options = FlatOptions::from_shared_options(&options);
let (tx, rx) = tokio::sync::mpsc::channel(1024);
let shared_context = Arc::new(TaskContext {
fs,
resolver,
plugin_driver,
options: Arc::clone(&options),
tx: tx.clone(),
meta: TaskContextMeta {
replace_global_define_config: if options.define.is_empty() {
None
} else {
ReplaceGlobalDefinesConfig::new(&options.define).map(Some).map_err(|errs| {
errs
.into_iter()
.map(|err| BuildDiagnostic::invalid_define_config(err.message.to_string()))
.collect::<Vec<BuildDiagnostic>>()
})?
},
},
});
let importers = std::mem::take(&mut cache.importers);
let mut intermediate_normal_modules = IntermediateNormalModules::new(is_full_scan, importers);
let runtime_id = intermediate_normal_modules.alloc_ecma_module_idx();
let remaining = if cache.module_id_to_idx.contains_key(RUNTIME_MODULE_KEY) {
intermediate_normal_modules.reset_ecma_module_idx();
0
} else {
let task = RuntimeModuleTask::new(runtime_id, Arc::clone(&shared_context), flat_options);
tokio::spawn(task.run());
cache.module_id_to_idx.insert(RUNTIME_MODULE_KEY.into(), VisitState::Seen(runtime_id));
1
};
let symbol_ref_db = SymbolRefDb::new(options.transform_options.is_jsx_preserve());
Ok(Self {
tx,
rx,
cache,
options,
remaining,
runtime_id,
is_full_scan,
shared_context,
symbol_ref_db,
intermediate_normal_modules,
new_added_modules_from_partial_scan: FxIndexSet::default(),
flat_options,
magic_string_tx,
})
}
#[expect(clippy::rc_buffer)]
fn try_spawn_new_task(
&mut self,
resolved_id: ResolvedId,
owner: Option<ModuleTaskOwner>,
is_user_defined_entry: bool,
assert_module_type: Option<ModuleType>,
user_defined_entries: Arc<Vec<(Option<ArcStr>, ResolvedId)>>,
) -> ModuleIdx {
let ctx = Arc::clone(&self.shared_context);
let idx = match self.cache.module_id_to_idx.get(&resolved_id.id) {
Some(VisitState::Seen(idx)) => return *idx,
Some(VisitState::Invalidate(idx)) => {
let idx = *idx;
self.intermediate_normal_modules.alloc_ecma_module_idx_sparse(idx);
self.cache.module_id_to_idx.insert(resolved_id.id.clone(), VisitState::Seen(idx));
idx
}
None if !self.is_full_scan => {
let len = self.cache.module_id_to_idx.len();
let idx = self.intermediate_normal_modules.alloc_ecma_module_idx_sparse(len.into());
self.new_added_modules_from_partial_scan.insert(idx);
self.cache.module_id_to_idx.insert(resolved_id.id.clone(), VisitState::Seen(idx));
idx
}
None => {
let idx = self.intermediate_normal_modules.alloc_ecma_module_idx();
self.cache.module_id_to_idx.insert(resolved_id.id.clone(), VisitState::Seen(idx));
idx
}
};
if resolved_id.external.is_external() {
let task = ExternalModuleTask::new(ctx, idx, resolved_id, user_defined_entries);
tokio::spawn(task.run().instrument(tracing::info_span!("external_module_task")));
} else {
let task = ModuleTask::new(
ctx,
idx,
resolved_id,
owner,
is_user_defined_entry,
assert_module_type,
self.flat_options,
self.magic_string_tx.clone(),
);
tokio::spawn(task.run().instrument(tracing::info_span!("normal_module_task")));
}
self.remaining += 1;
idx
}
#[tracing::instrument(level = "debug", skip_all)]
#[expect(clippy::too_many_lines)]
pub async fn fetch_modules(
&mut self,
fetch_mode: ScanMode<ResolvedId>,
) -> BuildResult<ModuleLoaderOutput> {
let mut errors = vec![];
let mut all_warnings: Vec<BuildDiagnostic> = vec![];
let user_defined_entries = match fetch_mode {
ScanMode::Full => {
resolve_user_defined_entries(
&self.options,
&self.shared_context.resolver,
&self.shared_context.plugin_driver,
)
.await?
}
ScanMode::Partial(_) => vec![],
};
let entries_count = user_defined_entries.len() + 1;
self.intermediate_normal_modules.modules.reserve(entries_count);
self.intermediate_normal_modules.index_ecma_ast.reserve(entries_count);
let mut entry_points = FxIndexSet::default();
let mut user_defined_entry_ids = FxHashSet::with_capacity(user_defined_entries.len());
let user_defined_entries = Arc::new(user_defined_entries);
for (defined_name, resolved_id) in user_defined_entries.iter() {
let idx = self.try_spawn_new_task(
resolved_id.clone(),
None,
true,
None,
Arc::clone(&user_defined_entries),
);
user_defined_entry_ids.insert(idx);
entry_points.insert(EntryPoint {
name: defined_name.clone(),
idx,
kind: EntryPointKind::UserDefined,
file_name: None,
related_stmt_infos: vec![],
});
}
if self.is_full_scan && self.options.experimental.is_incremental_build_enabled() {
self
.cache
.user_defined_entry
.extend(user_defined_entries.iter().map(|(_, resolved_id)| resolved_id.id.clone()));
}
for resolved_id in fetch_mode.iter() {
let resolved_id = resolved_id.clone();
self
.shared_context
.plugin_driver
.invalidate_context_load_module(&resolved_id.id.clone().into());
if let Entry::Occupied(mut occ) = self.cache.module_id_to_idx.entry(resolved_id.id.clone()) {
let idx = occ.get().idx();
occ.insert(VisitState::Invalidate(idx));
}
let is_user_defined_entry = self.cache.user_defined_entry.contains(&resolved_id.id);
self.try_spawn_new_task(
resolved_id,
None,
is_user_defined_entry,
None,
Arc::clone(&user_defined_entries),
);
}
let mut dynamic_import_entry_ids: FxHashMap<
ModuleIdx,
Vec<(ModuleIdx, StmtInfoIdx, ImportRecordIdx)>,
> = FxHashMap::default();
let mut dynamic_import_exports_usage_pairs = vec![];
let mut extra_entry_points = vec![];
let mut entry_point_to_reference_ids: FxHashMap<EntryPoint, Vec<ArcStr>> = FxHashMap::default();
let mut safely_merge_cjs_ns_map: FxHashMap<ModuleIdx, Vec<SymbolRef>> = FxHashMap::default();
let mut runtime_brief = None;
let mut overrode_preserve_entry_signature_map = FxHashMap::default();
while self.remaining > 0 {
let Some(msg) = self.rx.recv().await else {
break;
};
match msg {
ModuleLoaderMsg::NormalModuleDone(task_result) => {
let NormalModuleTaskResult {
mut module,
ecma_related: EcmaRelated { ast, symbols, mut dynamic_import_rec_exports_usage },
resolved_deps,
raw_import_records,
warnings,
} = *task_result;
all_warnings.extend(warnings);
let mut import_records = IndexVec::with_capacity(raw_import_records.len());
for ((rec_idx, mut raw_rec), resolved_id) in
raw_import_records.into_iter_enumerated().zip(resolved_deps)
{
if self.options.experimental.vite_mode.unwrap_or_default()
&& resolved_id.id.as_str().ends_with(".json")
{
raw_rec.meta.insert(ImportRecordMeta::JsonModule);
}
let idx = if let Some(idx) = self.try_spawn_with_cache(&resolved_id) {
idx
} else {
let normal_module = module.as_normal().unwrap();
let owner = ModuleTaskOwner::new(
normal_module.source.clone(),
normal_module.stable_id.as_str().into(),
raw_rec.span,
);
self.try_spawn_new_task(
resolved_id,
Some(owner),
false,
raw_rec.asserted_module_type.clone(),
Arc::clone(&user_defined_entries),
)
};
if raw_rec.meta.contains(ImportRecordMeta::SafelyMergeCjsNs) {
safely_merge_cjs_ns_map.entry(idx).or_default().push(raw_rec.namespace_ref);
}
self.intermediate_normal_modules.importers[idx].push(ImporterRecord {
kind: raw_rec.kind,
importer_path: ModuleId::new(module.id()),
importer_idx: module.idx(),
});
if let Some(usage) = dynamic_import_rec_exports_usage.remove(&rec_idx) {
dynamic_import_exports_usage_pairs.push((idx, usage));
}
if matches!(raw_rec.kind, ImportKind::DynamicImport)
&& !user_defined_entry_ids.contains(&idx)
{
match dynamic_import_entry_ids.entry(idx) {
Entry::Vacant(vac) => match raw_rec.related_stmt_info_idx {
Some(stmt_info_idx) => {
vac.insert(vec![(module.idx(), stmt_info_idx, rec_idx)]);
}
None => {
vac.insert(vec![]);
}
},
Entry::Occupied(mut occ) => {
if let Some(stmt_info_idx) = raw_rec.related_stmt_info_idx {
occ.get_mut().push((module.idx(), stmt_info_idx, rec_idx));
}
}
}
}
import_records.push(raw_rec.into_resolved(idx));
}
module.set_import_records(import_records);
let module_idx = module.idx();
if user_defined_entry_ids.contains(&module_idx) {
let normal_module = module.as_normal_mut().expect("should be normal module");
normal_module.is_user_defined_entry = true;
}
*self.intermediate_normal_modules.index_ecma_ast.get_mut(module_idx) = Some(ast);
*self.intermediate_normal_modules.modules.get_mut(module_idx) = Some(module);
self.symbol_ref_db.store_local_db(module_idx, symbols);
self.remaining -= 1;
}
ModuleLoaderMsg::ExternalModuleDone(task_result) => {
let ExternalModuleTaskResult {
id,
name,
idx,
identifier_name,
side_effects,
need_renormalize_render_path,
} = *task_result;
self.symbol_ref_db.store_local_db(
task_result.idx,
SymbolRefDbForModule::new(Scoping::default(), task_result.idx, ScopeId::new(0)),
);
let symbol_ref = self.symbol_ref_db.create_facade_root_symbol_ref(idx, &identifier_name);
let ext = ExternalModule::new(
idx,
id,
name,
identifier_name,
side_effects,
symbol_ref,
need_renormalize_render_path,
);
*self.intermediate_normal_modules.modules.get_mut(task_result.idx) = Some(ext.into());
self.remaining -= 1;
}
ModuleLoaderMsg::RuntimeNormalModuleDone(task_result) => {
let RuntimeModuleTaskResult {
local_symbol_ref_db,
mut module,
runtime,
ast,
raw_import_records,
resolved_deps,
} = *task_result;
let mut import_records = IndexVec::with_capacity(raw_import_records.len());
for ((rec_idx, raw_rec), info) in
raw_import_records.into_iter_enumerated().zip(resolved_deps)
{
let id = self.try_spawn_new_task(
info,
None,
false,
raw_rec.asserted_module_type.clone(),
Arc::clone(&user_defined_entries),
);
self.intermediate_normal_modules.importers[id].push(ImporterRecord {
kind: raw_rec.kind,
importer_path: module.id.clone(),
importer_idx: module.idx,
});
if matches!(raw_rec.kind, ImportKind::DynamicImport)
&& !user_defined_entry_ids.contains(&id)
{
match dynamic_import_entry_ids.entry(id) {
Entry::Vacant(vac) => match raw_rec.related_stmt_info_idx {
Some(stmt_info_idx) => {
vac.insert(vec![(module.idx, stmt_info_idx, rec_idx)]);
}
None => {
vac.insert(vec![]);
}
},
Entry::Occupied(mut occ) => {
if let Some(stmt_info_idx) = raw_rec.related_stmt_info_idx {
occ.get_mut().push((module.idx, stmt_info_idx, rec_idx));
}
}
}
}
import_records.push(raw_rec.into_resolved(id));
}
module.import_records = import_records;
*self.intermediate_normal_modules.modules.get_mut(self.runtime_id) = Some(module.into());
*self.intermediate_normal_modules.index_ecma_ast.get_mut(self.runtime_id) = Some(ast);
self.symbol_ref_db.store_local_db(self.runtime_id, local_symbol_ref_db);
runtime_brief = Some(runtime);
self.remaining -= 1;
}
ModuleLoaderMsg::FetchModule(resolve_id) => {
self.try_spawn_new_task(
*resolve_id,
None,
false,
None,
Arc::clone(&user_defined_entries),
);
}
ModuleLoaderMsg::AddEntryModule(msg) => {
let data = msg.chunk;
let result = load_entry_module(
&self.shared_context.resolver,
&self.shared_context.plugin_driver,
&data.id,
data.importer.as_deref(),
)
.await;
let resolved_id = match result {
Ok(result) => result,
Err(e) => {
errors.push(e);
continue;
}
};
let module_idx = self.try_spawn_new_task(
resolved_id,
None,
true,
None,
Arc::clone(&user_defined_entries),
);
if let Some(preserve_entry_signatures) = data.preserve_entry_signatures {
overrode_preserve_entry_signature_map.insert(module_idx, preserve_entry_signatures);
}
user_defined_entry_ids.insert(module_idx);
let entry = EntryPoint {
name: data.name.clone(),
idx: module_idx,
kind: EntryPointKind::EmittedUserDefined,
file_name: data.file_name.clone(),
related_stmt_infos: vec![],
};
entry_point_to_reference_ids
.entry(entry.clone())
.or_default()
.push(msg.reference_id.clone());
extra_entry_points.push(entry);
}
ModuleLoaderMsg::BuildErrors(e) => {
errors.extend(e);
self.remaining -= 1;
}
}
}
if !errors.is_empty() {
return Err(errors.into());
}
if let Some(tx) = self.magic_string_tx.as_ref() {
tx.send(SourceMapGenMsg::Terminate).expect(
"SourceMapGen: failed to send Terminate message - sourcemap worker thread died unexpectedly"
);
}
let dynamic_import_exports_usage_map = dynamic_import_exports_usage_pairs.into_iter().fold(
FxHashMap::default(),
|mut acc, (idx, usage)| {
match acc.entry(idx) {
Entry::Vacant(vac) => {
vac.insert(usage);
}
Entry::Occupied(mut occ) => {
occ.get_mut().merge(usage);
}
}
acc
},
);
let mut idx_of_module_info_need_update = vec![];
let is_dense_index_vec = self.intermediate_normal_modules.modules.is_index_vec();
let modules_iter = std::mem::take(&mut self.intermediate_normal_modules.modules)
.into_iter_enumerated()
.into_iter()
.map(|(idx, module)| {
let mut module = module.expect("Module tasks did't complete as expected");
if let Some(module) = module.as_normal_mut() {
let importers = &self.intermediate_normal_modules.importers[idx];
for importer in importers {
if importer.kind.is_static() {
module.importers.insert(importer.importer_path.clone());
module.importers_idx.insert(importer.importer_idx);
} else {
module.dynamic_importers.insert(importer.importer_path.clone());
}
}
if !importers.is_empty() {
idx_of_module_info_need_update.push(idx);
}
}
(idx, module)
});
let module_table = if is_dense_index_vec {
let vec = modules_iter.map(|(_, module)| module).collect();
HybridIndexVec::IndexVec(IndexVec::from_vec(vec))
} else {
let map = modules_iter.collect::<FxHashMap<_, _>>();
HybridIndexVec::Map(map)
};
idx_of_module_info_need_update.extend(extra_entry_points.iter().map(|item| item.idx));
idx_of_module_info_need_update.into_par_iter().for_each(|idx| {
let module = module_table.get(idx);
let Some(module) = module.as_normal() else {
return;
};
self
.shared_context
.plugin_driver
.set_module_info(&module.id, Arc::new(module.to_module_info(None)));
});
if !self.options.inline_dynamic_imports {
let dynamic_import_entry_ids = dynamic_import_entry_ids.into_iter().collect::<Vec<_>>();
entry_points.extend(dynamic_import_entry_ids.into_iter().map(|(idx, related_stmt_infos)| {
EntryPoint {
name: None,
idx,
kind: EntryPointKind::DynamicImport,
file_name: None,
related_stmt_infos,
}
}));
}
entry_points.extend(extra_entry_points);
if entry_points.is_empty() && self.is_full_scan {
Err(BuildDiagnostic::invalid_option(rolldown_error::InvalidOptionType::NoEntryPoint))?;
}
let entry_points = entry_points.into_iter().collect_vec();
let runtime = if self.is_full_scan {
tracing::debug!("changed_resolved_ids: {fetch_mode:#?}");
runtime_brief.expect("Failed to find runtime module. This should not happen")
} else {
RuntimeModuleBrief::dummy()
};
Ok(ModuleLoaderOutput {
runtime,
entry_points,
module_table,
warnings: all_warnings,
safely_merge_cjs_ns_map,
dynamic_import_exports_usage_map,
overrode_preserve_entry_signature_map,
entry_point_to_reference_ids,
symbol_ref_db: std::mem::take(&mut self.symbol_ref_db),
index_ecma_ast: std::mem::take(&mut self.intermediate_normal_modules.index_ecma_ast),
new_added_modules_from_partial_scan: std::mem::take(
&mut self.new_added_modules_from_partial_scan,
),
flat_options: self.flat_options,
})
}
fn try_spawn_with_cache(&self, resolved_dep: &ResolvedId) -> Option<ModuleIdx> {
if !self.options.experimental.is_incremental_build_enabled() {
return None;
}
self.cache.module_id_to_idx.get(&resolved_dep.id).map(|state| state.idx())
}
}