use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::sync::watch;
const OPTIMIZE_JS: &str = include_str!("assets/optimize-deps.mjs");
pub struct OptimizeView {
pub exclude: std::collections::HashSet<String>,
pub needs_interop: std::collections::HashSet<String>,
}
impl OptimizeView {
pub fn new(exclude: Vec<String>, needs_interop: Vec<String>) -> Self {
OptimizeView {
exclude: exclude.into_iter().collect(),
needs_interop: needs_interop.into_iter().collect(),
}
}
fn names(&self, set: &std::collections::HashSet<String>, path: &std::path::Path) -> bool {
package_name(path).is_some_and(|name| set.contains(&name))
}
pub fn is_excluded(&self, path: &std::path::Path) -> bool {
self.names(&self.exclude, path)
}
pub fn needs_forced_interop(&self, path: &std::path::Path) -> bool {
self.names(&self.needs_interop, path)
}
}
pub(crate) fn package_name(entry: &std::path::Path) -> Option<String> {
let comps: Vec<&std::ffi::OsStr> = entry.components().map(|c| c.as_os_str()).collect();
let idx = comps.iter().rposition(|c| *c == "node_modules")?;
let first = comps.get(idx + 1)?.to_str()?;
if first.starts_with('@') {
Some(format!("{first}/{}", comps.get(idx + 2)?.to_str()?))
} else {
Some(first.to_string())
}
}
#[derive(Clone)]
pub struct DepMeta {
pub file: String,
pub needs_interop: bool,
pub url: String,
pub file_hash: String,
}
fn url_version(url: &str) -> &str {
url.rsplit_once("?v=").map(|(_, v)| v).unwrap_or("")
}
fn flatten_dep_file(dep: &str) -> String {
let name: String = dep
.trim_start_matches('@')
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '_' || c == '.' || c == '-' {
c
} else {
'_'
}
})
.collect();
format!("{name}.mjs")
}
pub type DepMap = HashMap<String, DepMeta>;
pub struct OptimizedDeps {
rx: watch::Receiver<Option<Arc<DepMap>>>,
dir: PathBuf,
version: String,
rerun: Option<Arc<RerunState>>,
}
type OnCommit = Box<dyn Fn(&[String]) + Send + Sync>;
struct RerunState {
hash: String,
tx: watch::Sender<Option<Arc<DepMap>>>,
discovered: std::sync::Mutex<std::collections::BTreeMap<String, DepMeta>>,
pending: std::sync::Mutex<std::collections::HashSet<String>>,
wake: tokio::sync::mpsc::UnboundedSender<()>,
browser_version: std::sync::RwLock<String>,
failed: std::sync::Mutex<std::collections::HashSet<String>>,
dead: std::sync::atomic::AtomicBool,
exclude: Vec<String>,
discovery: bool,
on_commit: std::sync::OnceLock<OnCommit>,
}
pub enum DepServe {
Ready,
Outdated,
}
fn browser_hash<'a>(hash: &str, discovered: impl Iterator<Item = &'a String>) -> String {
let mut hasher = blake3::Hasher::new();
hasher.update(hash.as_bytes());
for dep in discovered {
hasher.update(b"\0");
hasher.update(dep.as_bytes());
}
hasher.finalize().to_hex()[..8].to_string()
}
fn module_list_contains(list: &[String], dep: &str) -> bool {
list.iter()
.any(|m| m == dep || dep.starts_with(&format!("{m}/")))
}
pub fn dep_url(file: &str, version: &str) -> String {
if version.is_empty() {
format!("/@oj-deps/{file}")
} else {
format!("/@oj-deps/{file}?v={version}")
}
}
impl OptimizedDeps {
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn version(&self) -> &str {
&self.version
}
pub async fn ready(&self) -> Arc<DepMap> {
let mut rx = self.rx.clone();
loop {
if let Some(map) = rx.borrow().clone() {
return map;
}
if rx.changed().await.is_err() {
return Arc::new(DepMap::new());
}
}
}
pub fn disabled() -> Self {
let (_tx, rx) = watch::channel(Some(Arc::new(DepMap::new())));
OptimizedDeps {
rx,
dir: PathBuf::new(),
version: String::new(),
rerun: None,
}
}
pub fn browser_version(&self) -> String {
match &self.rerun {
Some(r) => r.browser_version.read().unwrap().clone(),
None => self.version.clone(),
}
}
pub fn set_on_commit(&self, f: OnCommit) {
if let Some(r) = &self.rerun {
let _ = r.on_commit.set(f);
}
}
pub fn may_register(&self, dep: &str) -> bool {
self.rerun.as_ref().is_some_and(|r| {
r.discovery
&& !r.dead.load(std::sync::atomic::Ordering::Acquire)
&& !module_list_contains(&r.exclude, dep)
&& !r.failed.lock().unwrap().contains(dep)
})
}
pub fn meta_now(&self, dep: &str) -> Option<DepMeta> {
self.rx.borrow().as_ref().and_then(|m| m.get(dep).cloned())
}
pub fn register_missing(&self, dep: &str, needs_interop: bool) -> Option<DepMeta> {
let r = self.rerun.as_ref()?;
if !self.may_register(dep) {
return None;
}
if let Some(meta) = self.meta_now(dep) {
return Some(meta);
}
let meta = {
let mut discovered = r.discovered.lock().unwrap();
if let Some(meta) = discovered.get(dep) {
return Some(meta.clone());
}
let file = flatten_dep_file(dep);
discovered.insert(
dep.to_string(),
DepMeta {
file: file.clone(),
needs_interop,
url: String::new(),
file_hash: String::new(),
},
);
let version = browser_hash(&r.hash, discovered.keys());
let meta = DepMeta {
url: dep_url(&file, &version),
file,
needs_interop,
file_hash: String::new(),
};
discovered.insert(dep.to_string(), meta.clone());
meta
};
r.pending.lock().unwrap().insert(dep.to_string());
r.tx.send_modify(|cur| {
if let Some(map) = cur {
Arc::make_mut(map).insert(dep.to_string(), meta.clone());
}
});
let _ = r.wake.send(());
Some(meta)
}
pub async fn await_dep(&self, file: &str, req_version: Option<&str>) -> DepServe {
let deadline = tokio::time::Instant::now() + optimizer_timeout();
let mut rx = self.rx.clone();
loop {
enum St {
Wait,
Ready,
Outdated,
}
let st = {
let cur = rx.borrow_and_update();
match cur
.as_ref()
.and_then(|m| m.iter().find(|(_, meta)| meta.file == file))
{
None => St::Ready,
Some((dep, meta)) => {
let pending = self
.rerun
.as_ref()
.is_some_and(|r| r.pending.lock().unwrap().contains(dep));
if pending {
St::Wait
} else if req_version.is_some_and(|v| v != url_version(&meta.url)) {
St::Outdated
} else {
St::Ready
}
}
}
};
match st {
St::Ready => return DepServe::Ready,
St::Outdated => return DepServe::Outdated,
St::Wait => {
tokio::select! {
changed = rx.changed() => {
if changed.is_err() {
return DepServe::Ready;
}
}
_ = tokio::time::sleep_until(deadline) => return DepServe::Ready,
}
}
}
}
}
pub fn prepare(
root: &Path,
version: &str,
input: OptimizeInput,
host: Option<Arc<crate::plugins::PluginHost>>,
) -> Self {
let dir = oj_cache::cache_root(root).join("deps");
let hash = lockfile_hash(root, version, &input);
let short = hash[..8].to_string();
let (tx, rx) = watch::channel(None);
let cached = (!input.force).then(|| load_manifest(&dir, &hash)).flatten();
let (initial_discovered, initial_version) = match &cached {
Some(m) => {
let seeded = m
.discovered
.iter()
.filter_map(|d| m.map.get(d).map(|meta| (d.clone(), meta.clone())))
.collect();
(seeded, m.browser_hash.clone())
}
None => (std::collections::BTreeMap::new(), short.clone()),
};
let (wake, mut wake_rx) = tokio::sync::mpsc::unbounded_channel();
let rerun = Arc::new(RerunState {
hash: hash.clone(),
tx,
discovered: std::sync::Mutex::new(initial_discovered),
pending: std::sync::Mutex::new(std::collections::HashSet::new()),
wake,
browser_version: std::sync::RwLock::new(initial_version),
failed: std::sync::Mutex::new(std::collections::HashSet::new()),
dead: std::sync::atomic::AtomicBool::new(false),
exclude: input.exclude.clone(),
discovery: effective_auto_discover(input.no_discovery),
on_commit: std::sync::OnceLock::new(),
});
{
let rerun = Arc::clone(&rerun);
let root = root.to_path_buf();
let dir = dir.clone();
let cached_map = cached.map(|m| m.map);
tokio::spawn(async move {
match cached_map {
Some(map) => {
let _ = rerun.tx.send(Some(Arc::new(map)));
}
None => {
let map = match run_optimizer(
&root,
&dir,
&input,
host.as_deref(),
&Default::default(),
&hash[..8],
false,
)
.await
{
Some(map) => {
write_manifest(&dir, &hash, &hash[..8], [].iter(), &map);
map
}
None => {
rerun.dead.store(true, std::sync::atomic::Ordering::Release);
eprintln!(
"oj: dep pre-bundling unavailable; serving dependencies per-file"
);
DepMap::new()
}
};
let _ = rerun.tx.send(Some(Arc::new(map)));
}
}
while wake_rx.recv().await.is_some() {
debounce_wakes(&mut wake_rx, DEBOUNCE).await;
rerun_once(&rerun, &root, &dir, &hash, &input, host.as_deref()).await;
}
});
}
OptimizedDeps {
rx,
dir,
version: short,
rerun: Some(rerun),
}
}
}
const DEBOUNCE: std::time::Duration = std::time::Duration::from_millis(100);
async fn debounce_wakes(
rx: &mut tokio::sync::mpsc::UnboundedReceiver<()>,
window: std::time::Duration,
) {
loop {
match tokio::time::timeout(window, rx.recv()).await {
Ok(Some(())) => continue,
Ok(None) => return,
Err(_) => return,
}
}
}
async fn rerun_once(
rerun: &Arc<RerunState>,
root: &Path,
dir: &Path,
hash: &str,
input: &OptimizeInput,
host: Option<&crate::plugins::PluginHost>,
) {
let provisional = rerun.discovered.lock().unwrap().clone();
let pending: Vec<String> = rerun.pending.lock().unwrap().iter().cloned().collect();
if pending.is_empty() {
return;
}
let names: std::collections::BTreeSet<String> = provisional.keys().cloned().collect();
let candidate = browser_hash(hash, names.iter());
let Some(new_map) = run_optimizer(root, dir, input, host, &names, &candidate, true).await
else {
eprintln!("oj: re-optimizing newly discovered dependencies failed; serving them per-file");
fail_pending(rerun, &pending);
return;
};
let skipped: Vec<String> = pending
.iter()
.filter(|d| !new_map.contains_key(*d))
.cloned()
.collect();
if !skipped.is_empty() {
eprintln!(
"oj: {} could not be pre-bundled; serving per-file",
skipped.join(", ")
);
fail_pending(rerun, &skipped);
}
let late: std::collections::HashSet<String> = {
let p = rerun.pending.lock().unwrap();
p.iter()
.filter(|d| !pending.contains(*d))
.cloned()
.collect()
};
let old = rerun.tx.borrow().clone();
let old_map = old.as_deref();
let mut needs_reload = false;
if let Some(old_map) = old_map {
for (dep, old_meta) in old_map {
if pending.contains(dep) || late.contains(dep) {
continue;
}
match new_map.get(dep) {
Some(n) if !old_meta.file_hash.is_empty() && n.file_hash == old_meta.file_hash => {}
_ => {
needs_reload = true;
break;
}
}
}
}
let newly: Vec<String> = pending
.iter()
.filter(|d| new_map.contains_key(*d))
.cloned()
.collect();
for dep in &newly {
if provisional.get(dep).map(|p| p.needs_interop)
!= new_map.get(dep).map(|n| n.needs_interop)
{
needs_reload = true;
}
}
if needs_reload && !late.is_empty() {
let _ = rerun.wake.send(());
return;
}
{
let mut p = rerun.pending.lock().unwrap();
for dep in &pending {
p.remove(dep);
}
}
let final_map: DepMap = if needs_reload {
new_map
} else {
new_map
.into_iter()
.map(|(dep, mut meta)| {
let keep = old_map
.and_then(|m| m.get(&dep))
.map(|o| url_version(&o.url).to_string())
.unwrap_or_else(|| candidate.clone());
meta.url = dep_url(&meta.file, &keep);
(dep, meta)
})
.collect()
};
let bv = if needs_reload {
candidate.clone()
} else {
rerun.browser_version.read().unwrap().clone()
};
write_manifest(dir, hash, &bv, names.iter(), &final_map);
*rerun.browser_version.write().unwrap() = bv;
rerun.tx.send_modify(|cur| {
let prev = cur.take();
let still = rerun.pending.lock().unwrap();
*cur = Some(Arc::new(merge_pending(final_map, prev.as_deref(), &still)));
});
if needs_reload {
if let Some(cb) = rerun.on_commit.get() {
cb(&newly);
}
} else if !newly.is_empty() {
println!("oj: new dependencies optimized: {}", newly.join(", "));
}
}
fn merge_pending(
mut built: DepMap,
prev: Option<&DepMap>,
still_pending: &std::collections::HashSet<String>,
) -> DepMap {
if let Some(prev) = prev {
for dep in still_pending {
if !built.contains_key(dep) {
if let Some(meta) = prev.get(dep) {
built.insert(dep.clone(), meta.clone());
}
}
}
}
built
}
fn fail_pending(rerun: &Arc<RerunState>, deps: &[String]) {
{
let mut discovered = rerun.discovered.lock().unwrap();
let mut pending = rerun.pending.lock().unwrap();
let mut failed = rerun.failed.lock().unwrap();
for dep in deps {
discovered.remove(dep);
pending.remove(dep);
failed.insert(dep.clone());
}
}
rerun.tx.send_modify(|cur| {
if let Some(map) = cur {
let m = Arc::make_mut(map);
for dep in deps {
m.remove(dep);
}
}
});
if let Some(cb) = rerun.on_commit.get() {
cb(&[]);
}
}
fn effective_auto_discover(no_discovery: Option<bool>) -> bool {
!no_discovery.unwrap_or(false)
}
fn vendored_rolldown() -> Option<(&'static str, &'static str)> {
match oj_cache::start_bundle::vendored_rolldown() {
oj_cache::start_bundle::VendoredRolldown::Resolved { path, version } => {
Some((path.as_str(), version.as_str()))
}
_ => None,
}
}
#[derive(Default, Clone)]
pub struct OptimizeInput {
pub no_discovery: Option<bool>,
pub include: Vec<String>,
pub exclude: Vec<String>,
pub entries: Vec<String>,
pub build_inputs: Vec<String>,
pub build_out_dir: String,
pub dedupe: Vec<String>,
pub alias: Vec<(String, String)>,
pub force: bool,
pub bundler_options: Option<serde_json::Value>,
pub rolldown_options: Option<serde_json::Value>,
pub conditions: Vec<String>,
pub main_fields: Vec<String>,
pub extensions: Vec<String>,
pub preserve_symlinks: bool,
pub mode: String,
pub node_env: String,
pub plugin_names: Vec<String>,
pub needs_interop: Vec<String>,
}
pub fn optimizer_main_fields(config: &oj_config::OjConfig) -> Vec<String> {
oj_resolver::with_main_fallback(
oj_config::resolve_main_fields(config).unwrap_or_else(oj_resolver::default_main_fields),
)
}
const LOCKFILES: &[(&str, Option<&str>)] = &[
("node_modules/.pnpm/lock.yaml", None),
("node_modules/.package-lock.json", Some("patches")),
("node_modules/.yarn-state.yml", None),
("bun.lock", Some("patches")),
(".rush/temp/shrinkwrap-deps.json", None),
("aube-lock.yaml", None),
("nub.lock", Some("patches")),
(".pnp.cjs", Some(".yarn/patches")),
(".pnp.js", Some(".yarn/patches")),
("node_modules/.yarn-integrity", Some("patches")),
("bun.lockb", Some("patches")),
("package-lock.json", Some("patches")),
("yarn.lock", Some(".yarn/patches")),
("pnpm-lock.yaml", None),
("deno.lock", None),
];
fn hash_lockfiles(root: &Path, hasher: &mut blake3::Hasher) {
let mut dir = Some(root);
while let Some(d) = dir {
let mut found = false;
for (name, patches) in LOCKFILES {
if let Ok(bytes) = std::fs::read(d.join(name)) {
found = true;
hasher.update(name.as_bytes());
hasher.update(&bytes);
if let Some(patches) = patches {
if let Ok(meta) = std::fs::metadata(d.join(patches)) {
if meta.is_dir() {
if let Ok(mtime) = meta.modified() {
hasher.update(b"\0p");
hasher.update(format!("{mtime:?}").as_bytes());
}
}
}
}
}
}
if found {
return;
}
dir = d.parent();
}
}
fn hash_tagged_lists(hasher: &mut blake3::Hasher, lists: &[(&[u8], &Vec<String>)]) {
for (tag, list) in lists {
for entry in *list {
hasher.update(tag);
hasher.update(entry.as_bytes());
}
}
}
fn lockfile_hash(root: &Path, version: &str, input: &OptimizeInput) -> String {
let mut hasher = blake3::Hasher::new();
hasher.update(version.as_bytes());
hash_lockfiles(root, &mut hasher);
if let Ok(bytes) = std::fs::read(root.join("package.json")) {
hasher.update(b"package.json");
hasher.update(&bytes);
}
hasher.update(b"\0mode=");
hasher.update(input.mode.as_bytes());
hasher.update(b"\0node_env=");
hasher.update(input.node_env.as_bytes());
hash_tagged_lists(
&mut hasher,
&[
(b"\0i", &input.include),
(b"\0x", &input.exclude),
(b"\0e", &input.entries),
(b"\0b", &input.build_inputs),
(b"\0d", &input.dedupe),
(b"\0n", &input.needs_interop),
(b"\0p", &input.plugin_names),
],
);
hasher.update(b"\0bo=");
hasher.update(input.build_out_dir.as_bytes());
for (find, replacement) in &input.alias {
hasher.update(b"\0a");
hasher.update(find.as_bytes());
hasher.update(b"=");
hasher.update(replacement.as_bytes());
}
hasher.update(
format!(
"\0discovery:{}",
effective_auto_discover(input.no_discovery)
)
.as_bytes(),
);
if let Some((_, version)) = vendored_rolldown() {
hasher.update(format!("\0vendor:{version}").as_bytes());
}
if let Some(opts) = &input.bundler_options {
hasher.update(b"\0o");
hasher.update(opts.to_string().as_bytes());
}
hash_tagged_lists(
&mut hasher,
&[
(b"\0c", &input.conditions),
(b"\0m", &input.main_fields),
(b"\0t", &input.extensions),
],
);
hasher.update(&[b'\0', b's', input.preserve_symlinks as u8]);
hasher.finalize().to_hex().to_string()
}
fn parse_metadata(v: &serde_json::Value, version: &str) -> Option<DepMap> {
let obj = v.as_object()?;
let version = version.get(..8).unwrap_or(version);
let mut map = DepMap::new();
for (dep, meta) in obj {
let file = meta.get("file")?.as_str()?.to_string();
let needs_interop = meta
.get("needsInterop")
.and_then(serde_json::Value::as_bool)
.unwrap_or(true);
let v = meta.get("v").and_then(|v| v.as_str()).unwrap_or(version);
let file_hash = meta
.get("fileHash")
.and_then(|h| h.as_str())
.unwrap_or_default()
.to_string();
map.insert(
dep.clone(),
DepMeta {
url: dep_url(&file, v),
file,
needs_interop,
file_hash,
},
);
}
Some(map)
}
fn write_manifest<'a>(
dir: &Path,
hash: &str,
browser_version: &str,
discovered: impl Iterator<Item = &'a String>,
map: &DepMap,
) {
let metadata: serde_json::Map<String, serde_json::Value> = map
.iter()
.map(|(dep, m)| {
(
dep.clone(),
serde_json::json!({
"file": m.file,
"needsInterop": m.needs_interop,
"fileHash": m.file_hash,
"v": url_version(&m.url),
}),
)
})
.collect();
let manifest = serde_json::json!({
"hash": hash,
"browserHash": browser_version,
"discovered": discovered.collect::<Vec<_>>(),
"metadata": metadata,
});
let _ = std::fs::write(dir.join("manifest.json"), manifest.to_string());
}
struct Manifest {
map: DepMap,
discovered: std::collections::BTreeSet<String>,
browser_hash: String,
}
fn load_manifest(dir: &Path, hash: &str) -> Option<Manifest> {
let raw = std::fs::read_to_string(dir.join("manifest.json")).ok()?;
let v: serde_json::Value = serde_json::from_str(&raw).ok()?;
if v.get("hash")?.as_str()? != hash {
return None;
}
let browser_hash = v
.get("browserHash")
.and_then(|b| b.as_str())
.unwrap_or(hash.get(..8).unwrap_or(hash))
.to_string();
let map = parse_metadata(v.get("metadata")?, &browser_hash)?;
for m in map.values() {
if !dir.join(&m.file).exists() {
return None;
}
}
let discovered = v
.get("discovered")
.and_then(|d| d.as_array())
.map(|a| {
a.iter()
.filter_map(|v| v.as_str().map(String::from))
.collect()
})
.unwrap_or_default();
Some(Manifest {
map,
discovered,
browser_hash,
})
}
pub(crate) fn optimizer_timeout() -> std::time::Duration {
optimizer_timeout_from(oj_env::get().knobs.optimize_timeout.as_deref())
}
fn optimizer_timeout_from(raw: Option<&str>) -> std::time::Duration {
let secs = raw
.and_then(|v| v.trim().parse::<u64>().ok())
.filter(|s| *s > 0)
.unwrap_or(120);
std::time::Duration::from_secs(secs)
}
#[allow(clippy::too_many_arguments)]
async fn run_optimizer(
root: &Path,
dir: &Path,
input: &OptimizeInput,
host: Option<&crate::plugins::PluginHost>,
discovered: &std::collections::BTreeSet<String>,
browser_version: &str,
keep_existing: bool,
) -> Option<DepMap> {
let cache = oj_cache::cache_root(root);
std::fs::create_dir_all(&cache).ok()?;
crate::plugins::ensure_asset(&cache, "optimize-deps.mjs", OPTIMIZE_JS).ok()?;
let script = cache.join("optimize-deps.mjs");
let alias: Vec<[&str; 2]> = input
.alias
.iter()
.map(|(f, r)| [f.as_str(), r.as_str()])
.collect();
let auto_discover = effective_auto_discover(input.no_discovery);
let mut cfg = serde_json::json!({
"root": root.to_string_lossy(),
"outDir": dir.to_string_lossy(),
"entries": input.entries,
"buildInputs": input.build_inputs,
"buildOutDir": input.build_out_dir,
"include": input.include,
"implicitInclude": std::iter::once("react/jsx-dev-runtime".to_string())
.chain(discovered.iter().cloned())
.collect::<Vec<_>>(),
"keepExisting": keep_existing,
"exclude": input.exclude,
"dedupe": input.dedupe,
"alias": alias,
"needsInterop": input.needs_interop,
"autoDiscover": auto_discover,
"nodeEnv": input.node_env,
"esbuildOptions": input.bundler_options,
"rolldownOptions": input.rolldown_options,
"vendoredRolldown": vendored_rolldown().map(|(path, _)| path),
"resolve": {
"conditions": input.conditions,
"mainFields": input.main_fields,
"extensions": input.extensions,
"preserveSymlinks": input.preserve_symlinks,
},
});
if let Some(host) = host {
cfg["scanned"] = scan_through_plugins(host, &cfg).await;
}
let timeout = optimizer_timeout();
let job_root = root.to_path_buf();
let job_script = script.clone();
let result = tokio::task::spawn_blocking(move || {
crate::plugins::run_engine_job(&job_root, &job_script, "optimize", cfg, timeout)
})
.await
.ok()?;
let v = match result {
Ok(v) => v,
Err(oj_js::EngineError::Deadline) => {
eprintln!(
"oj: the dep pre-bundle did not finish within {}s and was stopped (raise OJ_OPTIMIZE_TIMEOUT for slower machines); deps are served unbundled",
timeout.as_secs()
);
return None;
}
Err(e) => {
eprintln!("oj: optimizer failed: {e}");
return None;
}
};
let metadata = v.get("metadata")?;
parse_metadata(metadata, browser_version)
}
async fn scan_through_plugins(
host: &crate::plugins::PluginHost,
cfg: &serde_json::Value,
) -> serde_json::Value {
match host.optimize_scan(&cfg.to_string()).await {
Ok(Some(json)) => serde_json::from_str(&json).unwrap_or(serde_json::Value::Null),
Ok(None) => serde_json::Value::Null,
Err(e) => {
eprintln!("oj: dependency scan through plugins failed ({e}); scanning without them");
serde_json::Value::Null
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn input(
include: &[&str],
exclude: &[&str],
entries: &[&str],
dedupe: &[&str],
) -> OptimizeInput {
OptimizeInput {
include: include.iter().map(|s| s.to_string()).collect(),
exclude: exclude.iter().map(|s| s.to_string()).collect(),
entries: entries.iter().map(|s| s.to_string()).collect(),
dedupe: dedupe.iter().map(|s| s.to_string()).collect(),
alias: Vec::new(),
..Default::default()
}
}
#[test]
fn optimizer_main_fields_append_vites_main_fallback() {
let from = |json: &str| -> oj_config::OjConfig { serde_json::from_str(json).unwrap() };
assert_eq!(
optimizer_main_fields(&from(
r#"{ "resolve": { "mainFields": ["browser", "module", "jsnext:main", "jsnext"] } }"#
)),
["browser", "module", "jsnext:main", "jsnext", "main"].map(String::from)
);
assert_eq!(
optimizer_main_fields(&from(
r#"{ "resolve": { "mainFields": ["main", "module"] } }"#
)),
["main", "module"].map(String::from)
);
assert_eq!(
optimizer_main_fields(&from("{}")),
oj_resolver::default_main_fields()
);
}
fn project(files: &[(&str, &str)]) -> tempfile::TempDir {
let dir = tempfile::tempdir().unwrap();
for (name, contents) in files {
std::fs::write(dir.path().join(name), contents).unwrap();
}
dir
}
#[test]
fn every_optimizer_list_is_distinguishable_in_the_key() {
let dir = project(&[("package.json", r#"{"name":"app"}"#)]);
let root = dir.path();
let key = |i: &OptimizeInput| lockfile_hash(root, "0.0.1", i);
let base = key(&input(&["react"], &[], &[], &[]));
assert_ne!(
base,
key(&input(&[], &["react"], &[], &[])),
"include vs exclude"
);
assert_ne!(
base,
key(&input(&[], &[], &["react"], &[])),
"include vs entries"
);
assert_ne!(
base,
key(&input(&[], &[], &[], &["react"])),
"include vs dedupe"
);
let forced = OptimizeInput {
include: vec!["react".into()],
needs_interop: vec!["react".into()],
..Default::default()
};
assert_ne!(base, key(&forced), "needsInterop is part of the key");
let with_plugin = OptimizeInput {
include: vec!["react".into()],
plugin_names: vec!["app-icons".into()],
..Default::default()
};
assert_ne!(base, key(&with_plugin), "plugin names are part of the key");
assert_ne!(
key(&input(&["a"], &["b"], &[], &[])),
key(&input(&["a", "b"], &[], &[], &[])),
"moving an entry across the include/exclude boundary"
);
assert_ne!(
key(&input(&[], &[], &["a", "b"], &[])),
key(&input(&[], &[], &["a"], &["b"])),
"moving an entry across the entries/dedupe boundary"
);
}
#[test]
fn the_key_covers_the_lockfiles_the_version_and_the_aliases() {
let dir = project(&[
(
"package.json",
r#"{"name":"app","dependencies":{"react":"18"}}"#,
),
("package-lock.json", r#"{"lockfileVersion":3}"#),
]);
let root = dir.path();
let empty = OptimizeInput::default();
let base = lockfile_hash(root, "0.0.1", &empty);
assert_eq!(base, lockfile_hash(root, "0.0.1", &empty), "deterministic");
assert_ne!(base, lockfile_hash(root, "0.0.2", &empty), "tool version");
std::fs::write(root.join("package-lock.json"), r#"{"lockfileVersion":4}"#).unwrap();
let after_lock = lockfile_hash(root, "0.0.1", &empty);
assert_ne!(base, after_lock, "a lockfile change must invalidate");
let dev = OptimizeInput {
mode: "development".into(),
..OptimizeInput::default()
};
let prod = OptimizeInput {
mode: "production".into(),
..OptimizeInput::default()
};
assert_ne!(
lockfile_hash(root, "0.0.1", &dev),
lockfile_hash(root, "0.0.1", &prod),
"mode"
);
let prod_env = OptimizeInput {
node_env: "production".into(),
..OptimizeInput::default()
};
assert_ne!(
lockfile_hash(root, "0.0.1", &empty),
lockfile_hash(root, "0.0.1", &prod_env),
"NODE_ENV"
);
let with_build_input = OptimizeInput {
build_inputs: vec!["src/main.ts".into()],
..OptimizeInput::default()
};
assert_ne!(
lockfile_hash(root, "0.0.1", &empty),
lockfile_hash(root, "0.0.1", &with_build_input),
"build inputs feed the scan entries"
);
let with_out_dir = OptimizeInput {
build_out_dir: "build".into(),
..OptimizeInput::default()
};
assert_ne!(
lockfile_hash(root, "0.0.1", &empty),
lockfile_hash(root, "0.0.1", &with_out_dir),
"build outDir shapes the html glob"
);
let aliased = OptimizeInput {
alias: vec![("~".into(), "./src".into())],
..OptimizeInput::default()
};
assert_ne!(after_lock, lockfile_hash(root, "0.0.1", &aliased), "alias");
let swapped = OptimizeInput {
alias: vec![("./src".into(), "~".into())],
..OptimizeInput::default()
};
assert_ne!(
lockfile_hash(root, "0.0.1", &aliased),
lockfile_hash(root, "0.0.1", &swapped),
"an alias is directional"
);
}
#[test]
fn every_vite_lockfile_format_and_the_patches_dir_key_the_prebundle() {
let dir = project(&[("package.json", r#"{"name":"app"}"#)]);
let root = dir.path();
let empty = OptimizeInput::default();
let key = || lockfile_hash(root, "v", &empty);
let bare = key();
std::fs::write(root.join("bun.lock"), "{}").unwrap();
let bun = key();
assert_ne!(bare, bun, "bun.lock");
std::fs::write(root.join("bun.lock"), "{\"a\":1}").unwrap();
assert_ne!(bun, key(), "bun.lock content");
std::fs::remove_file(root.join("bun.lock")).unwrap();
std::fs::write(root.join("deno.lock"), "{}").unwrap();
assert_ne!(bare, key(), "deno.lock");
std::fs::remove_file(root.join("deno.lock")).unwrap();
std::fs::create_dir_all(root.join("node_modules")).unwrap();
std::fs::write(root.join("node_modules/.package-lock.json"), "{}").unwrap();
let installed = key();
assert_ne!(bare, installed, "node_modules/.package-lock.json");
std::fs::create_dir_all(root.join("patches")).unwrap();
let patched = key();
assert_ne!(installed, patched, "patches dir");
assert_eq!(patched, key(), "deterministic while nothing changes");
}
#[test]
fn the_lockfile_is_looked_up_in_ancestor_directories() {
let dir = tempfile::tempdir().unwrap();
let app = dir.path().join("packages/app");
std::fs::create_dir_all(&app).unwrap();
std::fs::write(app.join("package.json"), r#"{"name":"app"}"#).unwrap();
let empty = OptimizeInput::default();
let before = lockfile_hash(&app, "v", &empty);
std::fs::write(dir.path().join("pnpm-lock.yaml"), "lockfileVersion: 9").unwrap();
let after = lockfile_hash(&app, "v", &empty);
assert_ne!(before, after, "ancestor lockfile is keyed");
std::fs::write(dir.path().join("pnpm-lock.yaml"), "lockfileVersion: 10").unwrap();
assert_ne!(
after,
lockfile_hash(&app, "v", &empty),
"and its content matters"
);
}
#[test]
fn a_manifest_is_only_reused_when_its_hash_and_its_files_are_there() {
let dir = tempfile::tempdir().unwrap();
let deps = dir.path().join("deps");
std::fs::create_dir_all(&deps).unwrap();
std::fs::write(deps.join("react.js"), "export default 1;").unwrap();
std::fs::write(
deps.join("manifest.json"),
r#"{"hash":"abc","metadata":{"react":{"file":"react.js","needsInterop":false}}}"#,
)
.unwrap();
let m = load_manifest(&deps, "abc").expect("matching hash loads");
assert_eq!(m.map["react"].file, "react.js");
assert_eq!(m.map["react"].url, "/@oj-deps/react.js?v=abc");
assert!(!m.map["react"].needs_interop);
assert!(
m.discovered.is_empty() && m.browser_hash == "abc",
"no rerun persisted"
);
std::fs::write(
deps.join("manifest.json"),
r#"{"hash":"abc","browserHash":"feedf00d","discovered":["latedep"],"metadata":{"react":{"file":"react.js","needsInterop":false}}}"#,
)
.unwrap();
let rerun = load_manifest(&deps, "abc").expect("rerun manifest loads");
assert_eq!(rerun.browser_hash, "feedf00d");
assert_eq!(rerun.map["react"].url, "/@oj-deps/react.js?v=feedf00d");
assert_eq!(
rerun.discovered.iter().cloned().collect::<Vec<_>>(),
["latedep"]
);
std::fs::write(
deps.join("manifest.json"),
r#"{"hash":"abc","metadata":{"react":{"file":"react.js","needsInterop":false}}}"#,
)
.unwrap();
assert!(load_manifest(&deps, "different").is_none(), "stale hash");
std::fs::remove_file(deps.join("react.js")).unwrap();
assert!(load_manifest(&deps, "abc").is_none(), "missing dep file");
}
#[test]
fn a_malformed_manifest_is_a_miss_not_a_panic() {
let dir = tempfile::tempdir().unwrap();
let deps = dir.path().join("deps");
std::fs::create_dir_all(&deps).unwrap();
for contents in [
"",
"{",
"null",
"[]",
r#"{"metadata":{}}"#,
r#"{"hash":"abc"}"#,
r#"{"hash":123,"metadata":{}}"#,
r#"{"hash":"abc","metadata":[]}"#,
r#"{"hash":"abc","metadata":{"react":{}}}"#,
r#"{"hash":"abc","metadata":{"react":{"file":42}}}"#,
] {
std::fs::write(deps.join("manifest.json"), contents).unwrap();
assert!(
load_manifest(&deps, "abc").is_none(),
"accepted {contents:?}"
);
}
std::fs::write(
deps.join("manifest.json"),
r#"{"hash":"abc","metadata":{}}"#,
)
.unwrap();
assert!(load_manifest(&deps, "abc")
.expect("empty is valid")
.map
.is_empty());
}
fn test_deps(discovery: bool, exclude: Vec<String>) -> OptimizedDeps {
let (tx, rx) = watch::channel(Some(Arc::new(DepMap::new())));
let (wake, _wake_rx) = tokio::sync::mpsc::unbounded_channel();
let rerun = Arc::new(RerunState {
hash: "testhash".into(),
tx,
discovered: std::sync::Mutex::new(std::collections::BTreeMap::new()),
pending: std::sync::Mutex::new(std::collections::HashSet::new()),
wake,
browser_version: std::sync::RwLock::new("00000000".into()),
failed: std::sync::Mutex::new(std::collections::HashSet::new()),
dead: std::sync::atomic::AtomicBool::new(false),
exclude,
discovery,
on_commit: std::sync::OnceLock::new(),
});
OptimizedDeps {
rx,
dir: PathBuf::new(),
version: "00000000".into(),
rerun: Some(rerun),
}
}
#[test]
fn registration_declines_excluded_failed_dead_and_no_discovery() {
let deps = test_deps(true, vec!["left-out".into()]);
assert!(deps.may_register("lodash"));
assert!(!deps.may_register("left-out"));
assert!(
!deps.may_register("left-out/sub"),
"exclude covers subpaths"
);
let r = deps.rerun.as_ref().unwrap();
r.failed.lock().unwrap().insert("broken".into());
assert!(deps.register_missing("broken", false).is_none());
r.dead.store(true, std::sync::atomic::Ordering::Release);
assert!(!deps.may_register("lodash"));
assert!(deps.register_missing("lodash", false).is_none());
assert!(deps.meta_now("lodash").is_none());
let off = test_deps(false, vec![]);
assert!(!off.may_register("lodash"), "noDiscovery declines all");
}
#[test]
fn registration_rewrites_to_the_future_url() {
let deps = test_deps(true, vec![]);
let meta = deps.register_missing("@scope/pkg", true).unwrap();
assert_eq!(meta.file, "scope_pkg.mjs");
assert!(meta.url.starts_with("/@oj-deps/scope_pkg.mjs?v="));
assert!(meta.needs_interop);
let live = deps.meta_now("@scope/pkg").expect("live map sees it");
assert_eq!(live.url, meta.url);
let again = deps.register_missing("@scope/pkg", true).unwrap();
assert_eq!(again.url, meta.url, "re-registration is idempotent");
}
#[tokio::test(start_paused = true)]
async fn debounce_slides_per_wake_and_drains_the_burst() {
let window = std::time::Duration::from_millis(100);
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
tx.send(()).unwrap();
rx.recv().await; let start = tokio::time::Instant::now();
let waiter = tokio::spawn(async move {
debounce_wakes(&mut rx, window).await;
(start.elapsed(), rx)
});
tokio::time::sleep(std::time::Duration::from_millis(60)).await;
tx.send(()).unwrap();
tokio::time::sleep(std::time::Duration::from_millis(60)).await;
tx.send(()).unwrap();
let (elapsed, mut rx) = waiter.await.unwrap();
assert_eq!(
elapsed,
std::time::Duration::from_millis(220),
"60 + 60, then one full quiet window"
);
assert!(
rx.try_recv().is_err(),
"every wake of the burst joined the one batch"
);
}
#[test]
fn merge_pending_carries_mid_rerun_registrations() {
let meta = |file: &str| DepMeta {
file: file.into(),
needs_interop: false,
url: dep_url(file, "aaaaaaaa"),
file_hash: String::new(),
};
let mut prev = DepMap::new();
prev.insert("early".into(), meta("early.mjs"));
prev.insert("late".into(), meta("late.mjs"));
let mut built = DepMap::new();
built.insert(
"early".into(),
DepMeta {
file_hash: "bb".into(),
..meta("early.mjs")
},
);
let still: std::collections::HashSet<String> = ["late".to_string()].into_iter().collect();
let out = merge_pending(built.clone(), Some(&prev), &still);
assert_eq!(
out["late"].url, prev["late"].url,
"a dep registered mid-rerun keeps its provisional entry"
);
assert_eq!(out["early"].file_hash, "bb", "bundled entries win");
assert!(
!merge_pending(built, None, &still).contains_key("late"),
"nothing to carry before the first commit"
);
}
#[test]
fn browser_hash_tracks_the_discovered_set() {
let set = |deps: &[&str]| -> std::collections::BTreeSet<String> {
deps.iter().map(|d| d.to_string()).collect()
};
let base = browser_hash("hash", set(&[]).iter());
let one = browser_hash("hash", set(&["lodash"]).iter());
let two = browser_hash("hash", set(&["lodash", "dayjs"]).iter());
assert_eq!(base.len(), 8);
assert_ne!(base, one, "a discovered dep bumps the version");
assert_ne!(one, two);
assert_eq!(two, browser_hash("hash", set(&["dayjs", "lodash"]).iter()));
}
#[test]
fn manifest_round_trips_per_dep_versions_and_file_hashes() {
let dir = tempfile::tempdir().unwrap();
let deps = dir.path().join("deps");
std::fs::create_dir_all(&deps).unwrap();
std::fs::write(deps.join("react.js"), "export default 1;").unwrap();
std::fs::write(deps.join("latedep.mjs"), "export default 2;").unwrap();
let mut map = DepMap::new();
map.insert(
"react".into(),
DepMeta {
file: "react.js".into(),
needs_interop: false,
url: dep_url("react.js", "11111111"),
file_hash: "aaaa".into(),
},
);
map.insert(
"latedep".into(),
DepMeta {
file: "latedep.mjs".into(),
needs_interop: true,
url: dep_url("latedep.mjs", "22222222"),
file_hash: "bbbb".into(),
},
);
let discovered = ["latedep".to_string()];
write_manifest(&deps, "abc", "11111111", discovered.iter(), &map);
let m = load_manifest(&deps, "abc").expect("round trip");
assert_eq!(m.browser_hash, "11111111");
assert_eq!(m.map["react"].url, "/@oj-deps/react.js?v=11111111");
assert_eq!(m.map["latedep"].url, "/@oj-deps/latedep.mjs?v=22222222");
assert_eq!(m.map["latedep"].file_hash, "bbbb");
assert!(m.map["latedep"].needs_interop);
assert_eq!(
m.discovered.iter().cloned().collect::<Vec<_>>(),
["latedep"]
);
}
#[test]
fn exclude_list_matches_deps_and_their_subpaths() {
let list = vec!["lodash".to_string(), "@scope/pkg".to_string()];
assert!(module_list_contains(&list, "lodash"));
assert!(module_list_contains(&list, "lodash/debounce"));
assert!(module_list_contains(&list, "@scope/pkg/sub"));
assert!(!module_list_contains(&list, "lodash-es"));
}
#[test]
fn needs_interop_defaults_to_true_when_the_optimizer_does_not_say() {
let v: serde_json::Value = serde_json::from_str(r#"{"dep":{"file":"dep.js"}}"#).unwrap();
let map = parse_metadata(&v, "0123456789abcdef").unwrap();
assert!(map["dep"].needs_interop);
}
#[test]
fn dep_urls_carry_the_short_prebundle_version() {
let v: serde_json::Value =
serde_json::from_str(r#"{"react":{"file":"react.js","needsInterop":false}}"#).unwrap();
let map = parse_metadata(&v, "0123456789abcdef0123").unwrap();
assert_eq!(map["react"].url, "/@oj-deps/react.js?v=01234567");
assert_eq!(map["react"].file, "react.js");
assert_eq!(
dep_url("x.js", ""),
"/@oj-deps/x.js",
"no version, no query"
);
let other = parse_metadata(&v, "fedcba9876543210").unwrap();
assert_ne!(other["react"].url, map["react"].url);
}
#[test]
fn optimizer_timeout_defaults_and_reads_env_seconds() {
assert_eq!(optimizer_timeout_from(None).as_secs(), 120);
assert_eq!(optimizer_timeout_from(Some("300")).as_secs(), 300);
assert_eq!(optimizer_timeout_from(Some(" 30 ")).as_secs(), 30);
assert_eq!(optimizer_timeout_from(Some("junk")).as_secs(), 120);
assert_eq!(optimizer_timeout_from(Some("0")).as_secs(), 120);
}
#[tokio::test]
async fn a_disabled_optimizer_is_ready_immediately_and_empty() {
let deps = OptimizedDeps::disabled();
assert!(deps.ready().await.is_empty());
assert_eq!(deps.dir(), Path::new(""));
}
#[test]
fn discovery_is_on_unless_no_discovery_is_set() {
assert!(effective_auto_discover(None));
assert!(effective_auto_discover(Some(false)));
assert!(!effective_auto_discover(Some(true)));
let dir = project(&[("package.json", r#"{"name":"app"}"#)]);
let key = |no_discovery| {
lockfile_hash(
dir.path(),
"v",
&OptimizeInput {
no_discovery,
..Default::default()
},
)
};
assert_eq!(
key(None),
key(Some(false)),
"unset is the discovering default"
);
assert_ne!(
key(None),
key(Some(true)),
"noDiscovery is its own prebundle"
);
}
#[test]
fn resolve_settings_change_the_prebundle_hash() {
let dir = std::env::temp_dir().join(format!("oj-opt-hash-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let base = OptimizeInput {
include: vec!["dep".into()],
conditions: vec!["browser".into(), "import".into()],
..Default::default()
};
let mut with_dev = base.clone();
with_dev.conditions.push("development".into());
let mut fields = base.clone();
fields.main_fields = vec!["main".into()];
let mut links = base.clone();
links.preserve_symlinks = true;
let h = |i: &OptimizeInput| lockfile_hash(&dir, "v", i);
assert_ne!(h(&base), h(&with_dev));
assert_ne!(h(&base), h(&fields));
assert_ne!(h(&base), h(&links));
assert_eq!(h(&base), h(&base.clone()));
let _ = std::fs::remove_dir_all(&dir);
}
}