use std::collections::{BTreeMap, BTreeSet};
use rayon::prelude::*;
use crate::config::{Config, DeployMode, ParsedSource, Refspec, SourceMode};
use crate::error::Result;
use crate::lock::{Lock, LockedSource, encode_ref, entry_matches, ref_discriminator};
use crate::projection::offer::OfferSelection;
use crate::source::{
Commit, MirrorKey, NormalizedUrl, RefreshPlan, ResolvePolicy, ResolveRequest, ResolvedRevision,
ResolvedSource, RevisionSpec, SnapshotId, SourceDirectoryEntry, SourceEntry, SourceError,
SourceInventory, SourceLocation, SourceName, SourcePath, SourceStore, digest_snapshot,
};
use super::progress::{FetchId, FetchOutcome, ProgressSink};
use super::{SourcePolicy, effective_protocol, remote_for};
type SourceResult<T> = std::result::Result<T, SourceError>;
pub struct RoutedSources {
pub locks: Vec<(String, LockedSource)>,
pub commits: BTreeMap<(String, String), String>,
pub resolved: ResolvedSourceMap,
}
pub type ResolvedSourceMap = BTreeMap<(String, String), ResolvedSource>;
#[derive(Debug, Clone)]
pub(super) struct ImportResolution {
pub source: String,
pub refspec: Refspec,
pub commit: Option<String>,
}
struct Unit {
name: String,
encoded_ref: String,
effective_ref: Refspec,
manifest_commit: Option<String>,
history: bool,
}
fn resolution_units(
config: &Config,
parsed: &BTreeMap<String, ParsedSource>,
imports: &[ImportResolution],
) -> Vec<Unit> {
let mut by_key: BTreeMap<(String, String), Unit> = BTreeMap::new();
let mut bound = BTreeSet::new();
let mut add =
|name: &str, effective_ref: Refspec, manifest_commit: Option<String>, history: bool| {
let encoded_ref = encode_ref(&effective_ref);
let earlier_history = by_key
.get(&(name.to_owned(), encoded_ref.clone()))
.is_some_and(|unit| unit.history);
by_key.insert(
(name.to_owned(), encoded_ref.clone()),
Unit {
name: name.to_owned(),
encoded_ref,
effective_ref,
manifest_commit,
history: history || earlier_history,
},
);
};
for target in config.targets.values() {
for binding in target.resolve_sources(parsed) {
bound.insert(binding.source.to_owned());
add(binding.source, binding.effective_ref, None, binding.history);
}
}
for import in imports {
bound.insert(import.source.clone());
add(
&import.source,
import.refspec.clone(),
import.commit.clone(),
false,
);
}
for (name, source) in parsed {
if !bound.contains(name) {
add(name, source.refspec(), None, false);
}
}
by_key
.into_values()
.filter(|unit| {
parsed
.get(&unit.name)
.is_none_or(|source| source.mode() != SourceMode::Build)
})
.collect()
}
fn refresh_plan(
parsed: &BTreeMap<String, ParsedSource>,
remotes: &BTreeMap<String, String>,
units: &[&Unit],
) -> Result<Option<RefreshPlan>> {
let Some(first) = units.first() else {
return Ok(None);
};
let Some(source) = parsed.get(&first.name) else {
return Ok(None);
};
if source.deploy_mode() == DeployMode::Link || source.mode() == SourceMode::Url {
return Ok(None);
}
let mut revisions = Vec::new();
for unit in units {
let revision = match &unit.manifest_commit {
Some(commit) => RevisionSpec::Commit(commit.parse::<Commit>()?),
None => revision_spec(&unit.effective_ref)?,
};
if !revisions.contains(&revision) {
revisions.push(revision);
}
}
Ok(Some(RefreshPlan {
location: SourceLocation::Git {
url: remote_for(remotes, &first.name)?.to_owned(),
},
revisions,
history: units.iter().any(|unit| unit.history),
}))
}
struct Resolved {
name: String,
encoded_ref: String,
commit: String,
locked: LockedSource,
source: ResolvedSource,
}
fn resolution_groups<'a>(
parsed: &BTreeMap<String, ParsedSource>,
remotes: &BTreeMap<String, String>,
units: &'a [Unit],
) -> Result<Vec<(String, Vec<&'a Unit>)>> {
let mut groups: BTreeMap<String, Vec<&Unit>> = BTreeMap::new();
for unit in units {
if !parsed.contains_key(&unit.name) {
continue;
}
let git = remote_for(remotes, &unit.name)?;
let key = MirrorKey::from_url(&NormalizedUrl::parse(git))
.as_str()
.to_owned();
groups.entry(key).or_default().push(unit);
}
Ok(groups.into_iter().collect())
}
fn revision_spec(refspec: &Refspec) -> Result<RevisionSpec> {
Ok(match refspec {
Refspec::Branch(branch) => RevisionSpec::Branch(branch.clone()),
Refspec::Tag(tag) => RevisionSpec::Tag(tag.clone()),
Refspec::Rev(commit) => RevisionSpec::Commit(commit.parse::<Commit>()?),
Refspec::Default => RevisionSpec::Default,
Refspec::None => RevisionSpec::None,
})
}
fn lock_hit<'l>(
config: &Config,
source: &ParsedSource,
unit: &Unit,
effective_lock: Option<&'l Lock>,
policy: SourcePolicy,
) -> Option<&'l LockedSource> {
if policy.refreshes() {
return None;
}
let discriminator = ref_discriminator(&unit.effective_ref, &source.refspec());
let protocol = effective_protocol(source, config);
effective_lock
.and_then(|l| l.find_entry(&unit.name, discriminator.as_deref()))
.filter(|l| entry_matches(source, &unit.effective_ref, l, &config.hosts, protocol))
}
#[expect(
clippy::too_many_arguments,
reason = "resolving one unit threads config/parsed/remotes/lock/backend plus the source policy"
)]
#[cfg_attr(feature = "trace", tracing::instrument(skip_all, fields(unit = %unit.name)))]
fn resolve_unit(
config: &Config,
parsed: &BTreeMap<String, ParsedSource>,
remotes: &BTreeMap<String, String>,
instances: &BTreeMap<String, String>,
effective_lock: Option<&Lock>,
store: &dyn SourceStore,
policy: SourcePolicy,
unit: &Unit,
mirror_refreshed: &mut bool,
) -> Result<Option<Resolved>> {
let Some(source) = parsed.get(&unit.name) else {
return Ok(None);
};
let git = remote_for(remotes, &unit.name)?;
let lock_entry = lock_hit(config, source, unit, effective_lock, policy);
if lock_entry.is_none() && policy.requires_pins() && source.deploy_mode() != DeployMode::Link {
let kind = if instances.contains_key(&unit.name) {
"transitive source"
} else {
"source"
};
return Err(policy.unpinned(&format!("{kind} `{}`", unit.name)));
}
let request = resolve_request(source, unit, git, lock_entry)?;
let source_resolution = if unit.manifest_commit.is_some() {
store.resolve(&request, ResolvePolicy::CachedOnly)?
} else {
resolve_source(
store,
&request,
source,
lock_entry,
policy,
mirror_refreshed,
)?
};
if source.deploy_mode() == DeployMode::Link {
if unit.encoded_ref != encode_ref(&source.refspec()) {
return Ok(None);
}
let commit = match &source_resolution.revision {
ResolvedRevision::WorktreeHead(Some(head)) => head.to_string(),
ResolvedRevision::WorktreeHead(None) => "link".to_owned(),
ResolvedRevision::Commit(_) => {
return Err(crate::error::Error::Source(format!(
"link source {} resolved to a git snapshot",
unit.name
)));
}
};
return Ok(Some(Resolved {
name: unit.name.clone(),
encoded_ref: unit.encoded_ref.clone(),
commit: commit.clone(),
locked: LockedSource {
name: unit.name.clone(),
git: git.to_owned(),
resolved: "link".to_owned(),
commit,
digest: "link:".to_owned(),
config_digest: source.config_digest(),
r#ref: None,
instance: instances.get(&unit.name).cloned(),
build: None,
},
source: source_resolution,
}));
}
let discriminator = ref_discriminator(&unit.effective_ref, &source.refspec());
let commit = match &source_resolution.revision {
ResolvedRevision::Commit(commit) => commit.to_string(),
ResolvedRevision::WorktreeHead(_) => {
return Err(crate::error::Error::Source(format!(
"copy source {} resolved to a worktree snapshot",
unit.name
)));
}
};
let digest = match lock_entry {
Some(locked)
if locked.commit == commit && locked.config_digest == source.config_digest() =>
{
prefetch_selected_leaves(store, &source_resolution, source)?;
locked.digest.clone()
}
_ => selected_source_digest(store, &source_resolution, source)?,
};
let resolved = if source.mode() == SourceMode::Url {
"url".to_owned()
} else {
unit.effective_ref.to_string()
};
Ok(Some(Resolved {
name: unit.name.clone(),
encoded_ref: unit.encoded_ref.clone(),
commit: commit.clone(),
locked: LockedSource {
name: unit.name.clone(),
git: git.to_owned(),
resolved,
commit,
digest,
config_digest: source.config_digest(),
r#ref: discriminator,
instance: instances.get(&unit.name).cloned(),
build: None,
},
source: source_resolution,
}))
}
fn resolve_request(
source: &ParsedSource,
unit: &Unit,
git: &str,
lock_entry: Option<&LockedSource>,
) -> Result<ResolveRequest> {
let revision = if let Some(commit) = &unit.manifest_commit {
RevisionSpec::Commit(commit.parse::<Commit>()?)
} else {
match lock_entry {
Some(locked) => RevisionSpec::Commit(locked.commit.parse::<Commit>()?),
None if source.deploy_mode() == DeployMode::Link => RevisionSpec::Default,
None => revision_spec(&unit.effective_ref)?,
}
};
let location = match source.deploy_mode() {
DeployMode::Link => SourceLocation::Worktree {
root: git.into(),
follow_symlinks: source.export_policy(false).allow_symlinks,
},
DeployMode::Copy if source.mode() == SourceMode::Url => SourceLocation::Url {
url: git.to_owned(),
},
DeployMode::Copy => SourceLocation::Git {
url: git.to_owned(),
},
};
Ok(ResolveRequest {
name: SourceName::trusted(unit.name.clone()),
location,
revision,
})
}
fn resolve_source(
store: &dyn SourceStore,
request: &ResolveRequest,
source: &ParsedSource,
lock_entry: Option<&LockedSource>,
policy: SourcePolicy,
mirror_refreshed: &mut bool,
) -> Result<ResolvedSource> {
if source.deploy_mode() == DeployMode::Link {
return store
.resolve(request, ResolvePolicy::CachedOnly)
.map_err(Into::into);
}
let resolved = if lock_entry.is_some() {
match store.resolve(request, ResolvePolicy::CachedOnly) {
Ok(resolved) => return Ok(resolved),
Err(error) if policy.offline() => return Err(error.into()),
Err(_) => store.resolve(request, ResolvePolicy::Refresh)?,
}
} else if source.mode() == SourceMode::Url || !*mirror_refreshed {
store.resolve(request, ResolvePolicy::Refresh)?
} else {
match store.resolve(request, ResolvePolicy::CachedOnly) {
Ok(resolved) => return Ok(resolved),
Err(_) => store.resolve(request, ResolvePolicy::Refresh)?,
}
};
*mirror_refreshed = true;
Ok(resolved)
}
#[cfg_attr(feature = "trace", tracing::instrument(skip_all))]
pub(super) fn selected_source_digest(
store: &dyn SourceStore,
resolved: &ResolvedSource,
source: &ParsedSource,
) -> Result<String> {
with_selected_leaves(store, resolved, source, |store, leaves| {
digest_snapshot(store, &resolved.snapshot, leaves).map_err(Into::into)
})
}
#[cfg_attr(feature = "trace", tracing::instrument(skip_all))]
fn prefetch_selected_leaves(
store: &dyn SourceStore,
resolved: &ResolvedSource,
source: &ParsedSource,
) -> Result<()> {
with_selected_leaves(store, resolved, source, |_, _| Ok(()))
}
fn with_selected_leaves<T, F>(
store: &dyn SourceStore,
resolved: &ResolvedSource,
source: &ParsedSource,
f: F,
) -> Result<T>
where
F: FnOnce(&dyn SourceStore, &[SourcePath]) -> Result<T>,
{
let inventory = store.inventory(&resolved.snapshot, None)?;
let root = (source.mode() != SourceMode::Url)
.then_some(source.root.as_deref())
.flatten();
let selection = OfferSelection::compile(
if source.mode() == SourceMode::Url {
&[]
} else {
source.includes()
},
if source.mode() == SourceMode::Url {
&[]
} else {
source.excludes()
},
root,
)?;
let candidates: Vec<&str> = inventory
.entries
.iter()
.map(|entry| entry.path.as_str())
.collect();
let leaves = selection
.select(&candidates)
.into_iter()
.map(|path| SourcePath::new(&path))
.collect::<std::result::Result<Vec<_>, _>>()?;
if let Some(root) = root {
let root = SourcePath::new(&root.to_string_lossy().replace('\\', "/"))?;
let rooted = RootedSnapshotStore { store, root };
rooted.prefetch(&resolved.snapshot, &leaves)?;
f(&rooted, &leaves)
} else {
store.prefetch(&resolved.snapshot, &leaves)?;
f(store, &leaves)
}
}
struct RootedSnapshotStore<'a> {
store: &'a dyn SourceStore,
root: SourcePath,
}
impl RootedSnapshotStore<'_> {
fn rooted(
&self,
path: &SourcePath,
) -> std::result::Result<SourcePath, crate::source::KernelError> {
SourcePath::new(&format!("{}/{}", self.root.as_str(), path.as_str()))
}
}
impl SourceStore for RootedSnapshotStore<'_> {
fn resolve(
&self,
request: &ResolveRequest,
policy: ResolvePolicy,
) -> SourceResult<ResolvedSource> {
self.store.resolve(request, policy)
}
fn inventory(
&self,
snapshot: &SnapshotId,
root: Option<&SourcePath>,
) -> SourceResult<SourceInventory> {
let root = match root {
Some(root) => self.rooted(root)?,
None => self.root.clone(),
};
self.store.inventory(snapshot, Some(&root))
}
fn read(&self, snapshot: &SnapshotId, path: &SourcePath) -> SourceResult<SourceEntry> {
let rooted = self.rooted(path)?;
let mut entry = self.store.read(snapshot, &rooted)?;
entry.meta.path = path.clone();
Ok(entry)
}
fn prefetch(&self, snapshot: &SnapshotId, leaves: &[SourcePath]) -> SourceResult<()> {
let rooted = leaves
.iter()
.map(|leaf| self.rooted(leaf))
.collect::<std::result::Result<Vec<_>, _>>()?;
self.store.prefetch(snapshot, &rooted)
}
fn list_directory(
&self,
snapshot: &SnapshotId,
path: Option<&SourcePath>,
) -> SourceResult<Vec<SourceDirectoryEntry>> {
let path = match path {
Some(path) => self.rooted(path)?,
None => self.root.clone(),
};
self.store.list_directory(snapshot, Some(&path))
}
}
fn default_thread_count(units: usize, cores: usize) -> usize {
units.min((2 * cores).max(50))
}
fn fetch_outcome<T>(resolved: &Result<T>, mirror_refreshed: bool) -> FetchOutcome {
match (resolved, mirror_refreshed) {
(Err(_), _) => FetchOutcome::Failed,
(Ok(_), true) => FetchOutcome::Fetched,
(Ok(_), false) => FetchOutcome::Cached,
}
}
#[expect(
clippy::too_many_arguments,
reason = "resolution threads config/parsed/remotes/lock/backend plus the source policy and jobs"
)]
#[cfg_attr(feature = "trace", tracing::instrument(skip_all))]
pub(super) fn resolve_sources(
config: &Config,
parsed: &BTreeMap<String, ParsedSource>,
remotes: &BTreeMap<String, String>,
instances: &BTreeMap<String, String>,
imports: &[ImportResolution],
effective_lock: Option<&Lock>,
store: &dyn SourceStore,
policy: SourcePolicy,
jobs: Option<usize>,
sink: &dyn ProgressSink,
) -> Result<RoutedSources> {
let units = resolution_units(config, parsed, imports);
let groups = resolution_groups(parsed, remotes, &units)?;
sink.resolve_planned(groups.len());
let cores = std::thread::available_parallelism().map_or(8, std::num::NonZero::get);
let threads = jobs
.unwrap_or_else(|| default_thread_count(units.len(), cores))
.max(1);
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(threads)
.build()
.map_err(|e| crate::error::Error::Source(e.to_string()))?;
pool.install(|| -> Result<RoutedSources> {
let resolved: Vec<Option<Resolved>> = groups
.into_par_iter()
.map(|(mirror, units)| {
let fetch = FetchId {
mirror,
sources: units.iter().map(|unit| unit.name.clone()).collect(),
};
sink.fetch_started(&fetch);
if let Some(plan) = refresh_plan(parsed, remotes, &units)? {
store.plan_refresh(&plan);
}
let mut mirror_refreshed = false;
let resolved = units
.into_iter()
.map(|unit| {
resolve_unit(
config,
parsed,
remotes,
instances,
effective_lock,
store,
policy,
unit,
&mut mirror_refreshed,
)
})
.collect::<Result<Vec<_>>>();
sink.fetch_finished(&fetch, fetch_outcome(&resolved, mirror_refreshed));
resolved
})
.collect::<Result<Vec<_>>>()?
.into_iter()
.flatten()
.collect();
let mut routed = Vec::new();
let mut resolved_commits = BTreeMap::new();
let mut resolved_sources = BTreeMap::new();
for entry in resolved.into_iter().flatten() {
resolved_commits.insert(
(entry.name.clone(), entry.encoded_ref),
entry.commit.clone(),
);
resolved_sources.insert((entry.name.clone(), entry.commit), entry.source);
routed.push((entry.name, entry.locked));
}
Ok(RoutedSources {
locks: routed,
commits: resolved_commits,
resolved: resolved_sources,
})
})
}
#[cfg(feature = "bench")]
pub fn resolve_sources_for_bench(
config: &Config,
parsed: &BTreeMap<String, ParsedSource>,
remotes: &BTreeMap<String, String>,
effective_lock: Option<&Lock>,
store: &dyn SourceStore,
policy: SourcePolicy,
jobs: Option<usize>,
) -> Result<RoutedSources> {
resolve_sources(
config,
parsed,
remotes,
&BTreeMap::new(),
&[],
effective_lock,
store,
policy,
jobs,
super::progress::SILENT,
)
}
#[cfg(test)]
mod tests {
use super::default_thread_count;
#[test]
fn default_thread_count_uses_one_thread_per_unit_below_cores() {
assert_eq!(default_thread_count(5, 8), 5);
}
#[test]
fn default_thread_count_uses_one_thread_per_unit_at_cores() {
assert_eq!(default_thread_count(8, 8), 8);
}
#[test]
fn default_thread_count_uses_one_thread_per_unit_between_cores_and_cap() {
assert_eq!(default_thread_count(12, 8), 12);
}
#[test]
fn default_thread_count_uses_one_thread_per_unit_at_twice_cores() {
assert_eq!(default_thread_count(16, 8), 16);
}
#[test]
fn default_thread_count_uses_one_thread_per_unit_up_to_floor() {
assert_eq!(default_thread_count(50, 8), 50);
}
#[test]
fn default_thread_count_caps_at_floor_when_twice_cores_is_below_it() {
assert_eq!(default_thread_count(60, 8), 50);
}
#[test]
fn default_thread_count_caps_at_twice_cores_when_above_floor() {
assert_eq!(default_thread_count(200, 32), 64);
}
#[test]
fn default_thread_count_single_core_single_unit() {
assert_eq!(default_thread_count(1, 1), 1);
}
#[test]
fn default_thread_count_single_core_uses_floor_not_cores() {
assert_eq!(default_thread_count(60, 1), 50);
}
#[test]
fn default_thread_count_zero_units_returns_zero() {
assert_eq!(default_thread_count(0, 8), 0);
}
}