use std::collections::BTreeMap;
use std::fs;
use std::io::Read;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
#[cfg(test)]
use std::cell::Cell;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::audit::{self, AuditOptions, AuditReport};
use crate::config::UserConfig;
use crate::error::{DaloError, DaloResult};
use crate::git;
use crate::inventory::{
self, InventoryWarning, InventoryWarningCode, SkillRecord, SourceInventory,
};
use crate::lockfile::{self, UserLock};
use crate::materialize::{self, MaterializationRollback, MaterializeOperationKind, SyncReport};
use crate::resolver::{self, Resolution, ResolvedSkill};
use crate::source::{self, SourceConfig, SourceKind};
use crate::store::{self, StorePaths};
pub const SOURCE_LOCK_SCHEMA_VERSION: u32 = 3;
const MIN_SUPPORTED_SOURCE_LOCK_SCHEMA_VERSION: u32 = 1;
const CATALOG_ADVANCE_SCHEMA_VERSION: u32 = 1;
#[cfg(test)]
thread_local! {
static SOURCE_LOCK_READ_COUNT: Cell<usize> = const { Cell::new(0) };
static INTERRUPT_AFTER_SELECTION_LOCK: Cell<bool> = const { Cell::new(false) };
}
#[cfg(test)]
pub(crate) fn reset_source_lock_read_count_for_test() {
SOURCE_LOCK_READ_COUNT.with(|count| count.set(0));
}
#[cfg(test)]
pub(crate) fn source_lock_read_count_for_test() -> usize {
SOURCE_LOCK_READ_COUNT.with(Cell::get)
}
#[cfg(test)]
pub(crate) fn interrupt_next_selection_after_lock_for_test() {
INTERRUPT_AFTER_SELECTION_LOCK.with(|interrupt| interrupt.set(true));
}
#[cfg(test)]
fn interrupt_after_selection_lock_if_armed() {
if INTERRUPT_AFTER_SELECTION_LOCK.with(|interrupt| interrupt.replace(false)) {
panic!("simulated interruption after writing source-lock.toml");
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct CatalogAdvanceJournal {
schema_version: u32,
source_id: String,
original_commit: String,
original_config: UserConfig,
original_source_lock: SourceLock,
original_user_lock: UserLock,
#[serde(default)]
completed: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SourceLock {
pub schema_version: u32,
#[serde(default)]
pub catalogs: Vec<CatalogLock>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CatalogLock {
pub source_id: String,
pub commit: String,
#[serde(default)]
pub selected: Vec<String>,
#[serde(default)]
pub inventory: Vec<CatalogEntry>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DirectCatalogPinMismatch {
pub source_id: String,
pub reason: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CatalogEntry {
#[serde(skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
pub slot_name: String,
pub path: String,
pub content_hash: String,
pub metadata_hash: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub requires: Vec<String>,
}
impl SourceLock {
#[must_use]
pub fn empty() -> Self {
Self {
schema_version: SOURCE_LOCK_SCHEMA_VERSION,
catalogs: Vec::new(),
}
}
#[must_use]
pub fn catalog(&self, source_id: &str) -> Option<&CatalogLock> {
self.catalogs.iter().find(|c| c.source_id == source_id)
}
}
pub fn direct_catalog_pin_mismatches(
paths: &StorePaths,
sources: &[SourceConfig],
) -> DaloResult<Vec<DirectCatalogPinMismatch>> {
let source_lock = read_source_lock(paths)?;
let mut mismatches = sources
.iter()
.filter(|source| {
source.enabled && source.kind == SourceKind::Catalog && source.declared_by.is_none()
})
.filter_map(|source| {
catalog_checkout_pin_mismatch(source, &source_lock).map(|reason| {
DirectCatalogPinMismatch {
source_id: source.id.clone(),
reason,
}
})
})
.collect::<Vec<_>>();
mismatches.sort_by(|left, right| left.source_id.cmp(&right.source_id));
Ok(mismatches)
}
#[must_use]
pub fn catalog_checkout_pin_mismatch(
source: &SourceConfig,
source_lock: &SourceLock,
) -> Option<String> {
catalog_checkout_pin_mismatch_with_head_cache(
source,
source_lock,
&mut source::SourceHeadCache::default(),
)
}
pub(crate) fn catalog_checkout_pin_mismatch_with_head_cache(
source: &SourceConfig,
source_lock: &SourceLock,
head_cache: &mut source::SourceHeadCache,
) -> Option<String> {
let Some(catalog_lock) = source_lock.catalog(&source.id) else {
return Some(
"source-lock.toml has no catalog pin; restore the catalog lock before syncing"
.to_owned(),
);
};
match head_cache.resolve_result(source) {
Ok(checkout) if checkout == catalog_lock.commit => None,
Ok(checkout) => Some(format!(
"checkout {} does not match source-lock pin {}; {}",
short_commit(&checkout),
short_commit(&catalog_lock.commit),
catalog_pin_recovery_guidance(source),
)),
Err(error) => Some(format!(
"checkout commit could not be read: {error}; {}",
catalog_pin_recovery_guidance(source)
)),
}
}
fn catalog_pin_recovery_guidance(source: &SourceConfig) -> String {
if source.declared_by.is_some() {
"restore the declaring team's `dalo.toml` if needed, then run `dalo sync` to recover the manifest-managed checkout".to_owned()
} else {
format!(
"restore the checkout to the pinned commit before syncing, or review `dalo source refresh {} --advance` after restoring it",
source.id
)
}
}
fn short_commit(commit: &str) -> &str {
commit.get(..12).unwrap_or(commit)
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CatalogCandidate {
pub id: Option<String>,
pub slot_name: String,
pub path: String,
pub description: Option<String>,
pub compatibility: Option<String>,
pub requires: Vec<String>,
pub selected: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CatalogInspectReport {
pub source_id: String,
pub candidates: Vec<CatalogCandidate>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CatalogSelectReport {
pub source_id: String,
pub added: Vec<String>,
pub removed: Vec<String>,
pub selected: Vec<String>,
pub dry_run: bool,
pub audits: Vec<AuditReport>,
pub migration_warnings: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub removed_binary_paths: Vec<PathBuf>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub binary_cleanup_warnings: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CatalogAddOutcome {
pub source: SourceConfig,
pub available_skills: Option<usize>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub inventory_warnings: Vec<InventoryWarning>,
pub dry_run: bool,
}
pub fn add_catalog_source(
paths: &StorePaths,
id: &str,
url: &str,
namespace: Option<&str>,
dry_run: bool,
) -> DaloResult<CatalogAddOutcome> {
if !source::is_valid_source_id(id) {
return Err(DaloError::InvalidSourceId {
id: id.to_owned(),
reason: source::SOURCE_ID_REQUIREMENTS.to_owned(),
});
}
let namespace = source::validate_source_namespace(namespace)?;
git::validate_remote_url(url)?;
let mut config = store::read_config(paths)?;
if config.sources.iter().any(|source| source.id == id) {
return Err(DaloError::SourceAlreadyExists {
source_id: id.to_owned(),
});
}
let checkout = paths.sources_dir.join(id).join("checkout");
let priority = config
.sources
.iter()
.map(|source| source.priority)
.max()
.unwrap_or(0)
+ 10;
let source = SourceConfig {
id: id.to_owned(),
kind: SourceKind::Catalog,
path: checkout.clone(),
subpath: None,
priority,
namespace,
enabled: true,
trusted: false,
url: Some(url.to_owned()),
branch: None,
update_policy: Some("pin".to_owned()),
selection: Vec::new(),
declared_by: None,
declared_ref: None,
};
if dry_run {
return Ok(CatalogAddOutcome {
source,
available_skills: None,
inventory_warnings: Vec::new(),
dry_run,
});
}
source::clone_source_checkout(url, &checkout)?;
let discovered = (|| -> DaloResult<(usize, Vec<InventoryWarning>)> {
let commit = git::rev_parse_head(&checkout)?;
let scan = scan_catalog(id, &checkout)?;
let inventory = catalog_inventory_from_scan(&checkout, &scan, &[])?;
let available_skills = inventory.len();
let mut lock = read_source_lock(paths)?;
lock.catalogs.retain(|c| c.source_id != id);
lock.catalogs.push(CatalogLock {
source_id: id.to_owned(),
commit,
selected: Vec::new(),
inventory,
});
lock.catalogs.sort_by(|a, b| a.source_id.cmp(&b.source_id));
write_source_lock(paths, &lock)?;
config.sources.push(source.clone());
source::sort_sources(&mut config.sources);
store::write_config(paths, &config)?;
Ok((available_skills, scan.warnings))
})();
let (available_skills, inventory_warnings) = discovered.inspect_err(|_| {
let _ = fs::remove_dir_all(&checkout);
if let Some(parent) = checkout.parent() {
let _ = fs::remove_dir(parent);
}
if let Ok(mut lock) = read_source_lock(paths) {
lock.catalogs.retain(|c| c.source_id != id);
let _ = write_source_lock(paths, &lock);
}
})?;
Ok(CatalogAddOutcome {
source,
available_skills: Some(available_skills),
inventory_warnings,
dry_run,
})
}
pub(crate) fn canonical_skill_selection(
paths: &StorePaths,
id: &str,
refs: &[String],
) -> DaloResult<Vec<String>> {
let source = catalog_source(paths, id)?;
let scan = scan_catalog(id, &source.path)?;
let candidates = catalog_candidates_from_scan(&source.path, &scan);
let mut selected = resolve_candidate_references(id, &candidates, refs)?
.iter()
.map(|candidate| canonical_selection(candidate, &candidates))
.collect::<Vec<_>>();
selected.sort();
selected.dedup();
Ok(selected)
}
pub(crate) fn resolve_catalog_skill_references(
source_id: &str,
checkout: &Path,
refs: &[String],
) -> DaloResult<Vec<CatalogCandidate>> {
let scan = scan_catalog(source_id, checkout)?;
let candidates = catalog_candidates_from_scan(checkout, &scan);
let mut selected = resolve_candidate_references(source_id, &candidates, refs)?;
selected.sort_by(|left, right| {
left.slot_name
.cmp(&right.slot_name)
.then(left.path.cmp(&right.path))
});
selected.dedup();
for candidate in &mut selected {
candidate.selected = true;
}
Ok(selected)
}
fn resolve_candidate_references(
source_id: &str,
candidates: &[CatalogCandidate],
refs: &[String],
) -> DaloResult<Vec<CatalogCandidate>> {
let resolved = refs
.iter()
.map(|reference| resolve_candidate_reference(source_id, candidates, reference))
.collect::<DaloResult<Vec<_>>>()?;
for candidate in &resolved {
if resolved
.iter()
.any(|other| other.slot_name == candidate.slot_name && other.path != candidate.path)
{
return Err(DaloError::StateError {
reason: format!(
"catalog `{source_id}` cannot select multiple paths for slot `{}`",
candidate.slot_name
),
});
}
}
Ok(resolved)
}
#[must_use = "the inspect report should be rendered"]
pub fn inspect_catalog(paths: &StorePaths, id: &str) -> DaloResult<CatalogInspectReport> {
let source = catalog_source(paths, id)?;
let selected = selection_set(&source.selection);
let scan = scan_catalog(id, &source.path)?;
let mut candidates = catalog_candidates_from_scan(&source.path, &scan);
for candidate in &mut candidates {
candidate.selected = candidate_is_selected(candidate, &selected);
}
Ok(CatalogInspectReport {
source_id: id.to_owned(),
candidates,
})
}
fn ensure_catalog_inventory_complete(inventory: &SourceInventory) -> DaloResult<()> {
if inventory.warnings.iter().any(|warning| {
warning.code == InventoryWarningCode::UnreadablePath
&& warning
.message
.starts_with("source inventory is incomplete:")
}) {
return Err(DaloError::StateError {
reason: "catalog inventory is incomplete because a source scan safety limit was reached; skills later in the tree may be missing. Reduce or split the source tree, then retry the command".to_owned(),
});
}
Ok(())
}
pub fn select_skills(
paths: &StorePaths,
id: &str,
refs: &[String],
unselect: bool,
dry_run: bool,
) -> DaloResult<CatalogSelectReport> {
let configured_source = catalog_source(paths, id)?;
if let Some(team) = &configured_source.declared_by {
return Err(DaloError::StateError {
reason: format!(
"catalog `{id}` is managed by `{team}`; edit `{}` in that team repository",
crate::team_manifest::TEAM_MANIFEST_FILE
),
});
}
let source_path = configured_source.path;
let original_lock = read_source_lock(paths)?;
if let Some(catalog_lock) = original_lock.catalog(id) {
ensure_catalog_checkout_matches_pin(id, &source_path, catalog_lock)?;
}
let scan = scan_catalog(id, &source_path)?;
let candidates = catalog_candidates_from_scan(&source_path, &scan);
let resolved = if unselect {
refs.iter()
.map(|reference| resolve_unselect_reference(id, &candidates, reference))
.collect::<DaloResult<Vec<_>>>()?
} else {
resolve_candidate_references(id, &candidates, refs)?
};
let audits = if unselect {
Vec::new()
} else {
resolved
.iter()
.map(|candidate| {
audit::audit_skill(
paths,
&format!("{id}:{}", candidate.slot_name),
&source_path.join(&candidate.path),
&audit::AuditOptions {
persist: !dry_run,
..audit::AuditOptions::default()
},
)
})
.collect::<DaloResult<Vec<_>>>()?
};
let mut lock = original_lock.clone();
let mut config = store::read_config(paths)?;
let known_sources = config
.sources
.iter()
.map(|candidate| candidate.id.clone())
.collect();
let source = config
.sources
.iter_mut()
.find(|source| source.id == id)
.ok_or_else(|| DaloError::unknown_source(id, known_sources))?;
let mut added = Vec::new();
let mut removed = Vec::new();
for (reference, candidate) in refs.iter().zip(&resolved) {
let unselect_slot = unselect
&& (reference == &candidate.slot_name
|| reference == &format!("{id}:{}", candidate.slot_name));
let matches_slot = |existing: &str| {
candidates.iter().any(|other| {
other.slot_name == candidate.slot_name
&& selection_matches_candidate(existing, other)
})
};
let was_selected = source.selection.iter().any(|existing| {
selection_matches_candidate(existing, candidate)
|| (unselect_slot && matches_slot(existing))
});
source.selection.retain(|existing| {
!selection_matches_candidate(existing, candidate)
&& (unselect && !unselect_slot || !matches_slot(existing))
});
if !unselect {
source
.selection
.push(canonical_selection(candidate, &candidates));
}
match (unselect, was_selected) {
(true, true) => removed.push(reference.clone()),
(false, false) => added.push(reference.clone()),
_ => {}
}
}
source.selection.sort();
source.selection.dedup();
added.sort();
added.dedup();
removed.sort();
removed.dedup();
let selected = source.selection.clone();
let mut migration_warnings = Vec::new();
if !dry_run {
migration_warnings = migrate_source_lock_inventories(&mut lock, &config.sources, Some(id))?;
let commit = git::rev_parse_head(&source_path)?;
let inventory = if let Some(index) = lock
.catalogs
.iter()
.position(|catalog| catalog.source_id == id)
{
if lock.schema_version == SOURCE_LOCK_SCHEMA_VERSION
&& lock.catalogs[index].commit == commit
{
let mut inventory = lock.catalogs[index].inventory.clone();
let missing = scan
.skills
.iter()
.filter(|skill| {
!inventory
.iter()
.any(|entry| entry.path == relative_path(&source_path, &skill.path))
})
.cloned()
.collect();
let mut missing_scan = scan.clone();
missing_scan.skills = missing;
inventory.extend(catalog_inventory_from_scan(
&source_path,
&missing_scan,
&selected,
)?);
inventory.sort_by(|a, b| a.slot_name.cmp(&b.slot_name).then(a.path.cmp(&b.path)));
hydrate_selected_content_hashes(&mut inventory, &source_path, &scan, &selected)?;
inventory
} else {
catalog_inventory_from_scan(&source_path, &scan, &selected)?
}
} else {
catalog_inventory_from_scan(&source_path, &scan, &selected)?
};
if let Some(index) = lock
.catalogs
.iter()
.position(|catalog| catalog.source_id == id)
{
lock.catalogs[index].selected = selected.clone();
lock.catalogs[index].commit = commit;
lock.catalogs[index].inventory = inventory;
} else {
lock.catalogs.push(CatalogLock {
source_id: id.to_owned(),
commit,
selected: selected.clone(),
inventory,
});
lock.catalogs.sort_by(|a, b| a.source_id.cmp(&b.source_id));
}
write_source_lock(paths, &lock)?;
#[cfg(test)]
interrupt_after_selection_lock_if_armed();
if let Err(error) = store::write_config(paths, &config) {
if let Err(rollback) = write_source_lock(paths, &original_lock) {
return Err(DaloError::Io(std::io::Error::other(format!(
"{error}; additionally failed to roll back source lock: {rollback}"
))));
}
return Err(error);
}
}
Ok(CatalogSelectReport {
source_id: id.to_owned(),
added,
removed,
selected,
dry_run,
audits,
migration_warnings,
removed_binary_paths: Vec::new(),
binary_cleanup_warnings: Vec::new(),
})
}
#[must_use]
pub fn skill_is_selected(skill: &SkillRecord, selection: &[String], source_root: &Path) -> bool {
selection.iter().any(|reference| {
reference == &skill.slot_name
|| reference == &skill.source_ref
|| skill.id.as_deref() == Some(reference.as_str())
|| skill
.path
.strip_prefix(source_root)
.is_ok_and(|path| path.to_string_lossy() == reference.as_str())
})
}
pub fn read_source_lock(paths: &StorePaths) -> DaloResult<SourceLock> {
#[cfg(test)]
SOURCE_LOCK_READ_COUNT.with(|count| count.set(count.get() + 1));
match fs::symlink_metadata(&paths.source_lock_file) {
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok(SourceLock::empty());
}
Err(error) => return Err(error.into()),
}
let content = fs::read_to_string(&paths.source_lock_file)?;
let lock: SourceLock = toml::from_str(&content).map_err(|error| DaloError::FileParse {
path: paths.source_lock_file.clone(),
reason: error.to_string(),
})?;
if !(MIN_SUPPORTED_SOURCE_LOCK_SCHEMA_VERSION..=SOURCE_LOCK_SCHEMA_VERSION)
.contains(&lock.schema_version)
{
return Err(DaloError::UnsupportedSchema {
path: paths.source_lock_file.clone(),
version: lock.schema_version,
supported: SOURCE_LOCK_SCHEMA_VERSION,
});
}
Ok(lock)
}
pub fn write_source_lock(paths: &StorePaths, lock: &SourceLock) -> DaloResult<()> {
store::write_toml_atomic(&paths.source_lock_file, lock)
}
pub fn catalog_inventory(checkout: &Path, selection: &[String]) -> DaloResult<Vec<CatalogEntry>> {
let inventory = scan_catalog("catalog", checkout)?;
catalog_inventory_from_scan(checkout, &inventory, selection)
}
fn scan_catalog(source_id: &str, checkout: &Path) -> DaloResult<SourceInventory> {
let inventory = inventory::scan_catalog_source(source_id, checkout)?;
ensure_catalog_inventory_complete(&inventory)?;
Ok(inventory)
}
fn catalog_inventory_from_scan(
checkout: &Path,
inventory: &SourceInventory,
selection: &[String],
) -> DaloResult<Vec<CatalogEntry>> {
let mut entries = Vec::with_capacity(inventory.skills.len());
for skill in &inventory.skills {
entries.push(CatalogEntry {
id: skill.id.clone(),
slot_name: skill.slot_name.clone(),
path: relative_path(checkout, &skill.path),
content_hash: skill_is_selected(skill, selection, checkout)
.then(|| hash_skill_directory(checkout, &skill.path))
.transpose()?
.unwrap_or_default(),
metadata_hash: hash_metadata(skill),
requires: skill.requires.clone(),
});
}
entries.sort_by(|a, b| a.slot_name.cmp(&b.slot_name).then(a.path.cmp(&b.path)));
Ok(entries)
}
fn hydrate_selected_content_hashes(
entries: &mut [CatalogEntry],
checkout: &Path,
inventory: &SourceInventory,
selection: &[String],
) -> DaloResult<()> {
for entry in entries {
let Some(skill) = inventory
.skills
.iter()
.find(|skill| relative_path(checkout, &skill.path) == entry.path)
else {
continue;
};
if skill_is_selected(skill, selection, checkout) && entry.content_hash.is_empty() {
entry.content_hash = hash_skill_directory(checkout, &skill.path)?;
}
}
Ok(())
}
fn catalog_candidates_from_scan(
checkout: &Path,
inventory: &SourceInventory,
) -> Vec<CatalogCandidate> {
let mut candidates = inventory
.skills
.iter()
.map(|skill| CatalogCandidate {
id: skill.id.clone(),
slot_name: skill.slot_name.clone(),
path: relative_path(checkout, &skill.path),
description: skill.description.clone(),
compatibility: skill.compatibility.clone(),
requires: skill.requires.clone(),
selected: false,
})
.collect::<Vec<_>>();
candidates.sort_by(|a, b| a.slot_name.cmp(&b.slot_name).then(a.path.cmp(&b.path)));
candidates
}
fn catalog_source(paths: &StorePaths, id: &str) -> DaloResult<SourceConfig> {
let config = store::read_config(paths)?;
catalog_source_from_config(&config.sources, id)
}
fn catalog_source_from_config(sources: &[SourceConfig], id: &str) -> DaloResult<SourceConfig> {
let known_sources = sources
.iter()
.map(|candidate| candidate.id.clone())
.collect();
let source = sources
.iter()
.find(|source| source.id == id)
.cloned()
.ok_or_else(|| DaloError::unknown_source(id, known_sources))?;
if source.kind != SourceKind::Catalog {
return Err(DaloError::NotACatalogSource {
source_id: id.to_owned(),
});
}
Ok(source)
}
fn selection_set(selection: &[String]) -> Vec<String> {
selection.to_vec()
}
fn candidate_is_selected(candidate: &CatalogCandidate, selection: &[String]) -> bool {
selection
.iter()
.any(|reference| candidate_matches_ref(candidate, reference))
}
fn candidate_matches_ref(candidate: &CatalogCandidate, reference: &str) -> bool {
candidate.slot_name == reference
|| candidate.path == reference
|| candidate.id.as_deref() == Some(reference)
}
fn resolve_candidate_reference(
source_id: &str,
candidates: &[CatalogCandidate],
reference: &str,
) -> DaloResult<CatalogCandidate> {
let source_prefix = format!("{source_id}:");
let lookup = reference.strip_prefix(&source_prefix).unwrap_or(reference);
let mut matches = candidates
.iter()
.filter(|candidate| candidate_matches_ref(candidate, reference))
.cloned()
.collect::<Vec<_>>();
if lookup != reference {
for candidate in candidates
.iter()
.filter(|candidate| candidate_matches_ref(candidate, lookup))
{
if !matches.contains(candidate) {
matches.push(candidate.clone());
}
}
}
match matches.as_slice() {
[] => Err(DaloError::skill_not_found_for_reference(
format!("{source_id}:{lookup}"),
lookup,
candidates
.iter()
.map(|candidate| candidate.slot_name.clone())
.collect(),
format!("dalo source inspect {source_id}"),
)),
[candidate] => Ok(candidate.clone()),
_ => Err(DaloError::AmbiguousSkillReference {
reference: reference.to_owned(),
matches: matches
.iter()
.map(candidate_display)
.collect::<Vec<_>>()
.join(", "),
}),
}
}
fn resolve_unselect_reference(
source_id: &str,
candidates: &[CatalogCandidate],
reference: &str,
) -> DaloResult<CatalogCandidate> {
let source_prefix = format!("{source_id}:");
let lookup = reference.strip_prefix(&source_prefix).unwrap_or(reference);
if let Some(candidate) = candidates
.iter()
.find(|candidate| candidate.slot_name == lookup)
{
return Ok(candidate.clone());
}
resolve_candidate_reference(source_id, candidates, reference)
}
fn canonical_selection(candidate: &CatalogCandidate, candidates: &[CatalogCandidate]) -> String {
candidate
.id
.clone()
.filter(|id| {
candidates.iter().all(|other| {
other.path == candidate.path
|| (other.id.as_ref() != Some(id)
&& other.slot_name != *id
&& other.path != *id
&& other.slot_name != candidate.slot_name)
})
})
.unwrap_or_else(|| {
if candidate.path.is_empty() || candidate.path == "." {
candidate.slot_name.clone()
} else {
candidate.path.clone()
}
})
}
fn selection_matches_candidate(selection: &str, candidate: &CatalogCandidate) -> bool {
selection == candidate.slot_name
|| selection == candidate.path
|| candidate.id.as_deref() == Some(selection)
}
fn candidate_display(candidate: &CatalogCandidate) -> String {
match &candidate.id {
Some(id) => format!("{id} ({})", candidate.path),
None => candidate.path.clone(),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DriftCode {
NewAvailable,
SelectedChanged,
SelectedMoved,
SelectedRemoved,
}
impl DriftCode {
#[must_use]
pub fn blocks_sync(self) -> bool {
matches!(self, Self::SelectedRemoved)
}
#[must_use]
pub fn as_str(self) -> &'static str {
match self {
Self::NewAvailable => "new_available",
Self::SelectedChanged => "selected_changed",
Self::SelectedMoved => "selected_moved",
Self::SelectedRemoved => "selected_removed",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DriftEntry {
pub code: DriftCode,
pub skill: String,
pub message: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CatalogDrift {
pub source_id: String,
pub pinned_commit: String,
pub upstream_commit: String,
pub outcomes: Vec<DriftEntry>,
pub migration_warnings: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CatalogAdvanceReport {
pub source_id: String,
pub old_lock: CatalogLock,
pub new_lock: CatalogLock,
pub outcomes: Vec<DriftEntry>,
pub selection_before: Vec<String>,
pub selection_after: Vec<String>,
pub audits: Vec<AuditReport>,
pub sync: SyncReport,
pub blocking_reasons: Vec<String>,
pub migration_warnings: Vec<String>,
pub dry_run: bool,
pub advanced: bool,
}
#[must_use]
pub fn compare_catalog_inventory(
locked: &[CatalogEntry],
selected: &[String],
fresh: &[CatalogEntry],
) -> Vec<DriftEntry> {
let is_selected = |entry: &CatalogEntry| {
selected.iter().any(|reference| {
reference == &entry.slot_name
|| reference == &entry.path
|| entry.id.as_deref() == Some(reference.as_str())
})
};
let mut outcomes = Vec::new();
for fresh_entry in fresh {
let known = locked.iter().any(|l| same_entry(l, fresh_entry));
if !known && !is_selected(fresh_entry) {
outcomes.push(DriftEntry {
code: DriftCode::NewAvailable,
skill: fresh_entry.slot_name.clone(),
message: format!(
"`{}` is newly available (not selected)",
fresh_entry.slot_name
),
});
}
}
for locked_entry in locked.iter().filter(|entry| is_selected(entry)) {
let by_id = locked_entry
.id
.as_deref()
.filter(|id| unique_entry_with_id(locked, id).is_some())
.and_then(|id| unique_entry_with_id(fresh, id));
let by_path = fresh.iter().find(|f| f.path == locked_entry.path);
let skill = locked_entry
.id
.clone()
.unwrap_or_else(|| locked_entry.slot_name.clone());
match by_id.or(by_path) {
Some(fresh_entry) => {
let moved = by_id.is_some() && fresh_entry.path != locked_entry.path;
let changed = fresh_entry.content_hash != locked_entry.content_hash
|| fresh_entry.metadata_hash != locked_entry.metadata_hash;
if moved {
outcomes.push(DriftEntry {
code: DriftCode::SelectedMoved,
skill: skill.clone(),
message: format!(
"`{}` moved from `{}` to `{}`",
locked_entry.slot_name, locked_entry.path, fresh_entry.path
),
});
}
if changed {
outcomes.push(DriftEntry {
code: DriftCode::SelectedChanged,
skill,
message: format!("`{}` changed upstream", locked_entry.slot_name),
});
}
}
None => outcomes.push(DriftEntry {
code: DriftCode::SelectedRemoved,
skill,
message: format!("`{}` was removed upstream", locked_entry.slot_name),
}),
}
}
outcomes.sort_by(|a, b| {
a.code
.as_str()
.cmp(b.code.as_str())
.then_with(|| a.skill.cmp(&b.skill))
});
outcomes
}
pub fn check_catalog_drift(paths: &StorePaths, id: &str) -> DaloResult<CatalogDrift> {
let _catalog_lock = store::CatalogLock::acquire_shared(paths)?;
ensure_no_pending_catalog_advance(paths)?;
let config = store::read_config(paths)?;
let source = catalog_source_from_config(&config.sources, id)?;
let mut lock = read_source_lock(paths)?;
let migration_warnings = if lock.schema_version != SOURCE_LOCK_SCHEMA_VERSION {
migrate_source_lock_inventories(&mut lock, &config.sources, Some(id))?
} else {
Vec::new()
};
let catalog_lock = lock
.catalog(id)
.ok_or_else(|| DaloError::unknown_source(id, Vec::new()))?
.clone();
if source.declared_by.is_some() {
return Ok(CatalogDrift {
source_id: id.to_owned(),
pinned_commit: catalog_lock.commit.clone(),
upstream_commit: catalog_lock.commit,
outcomes: Vec::new(),
migration_warnings,
});
}
ensure_catalog_checkout(id, &source.path)?;
git::fetch(&source.path)?;
git::prune_worktrees(&source.path)?;
let upstream_commit = git::rev_parse(&source.path, "FETCH_HEAD")?;
let fresh = if upstream_commit == catalog_lock.commit {
catalog_lock.inventory.clone()
} else {
let temp = tempfile::tempdir()?;
let worktree = temp.path().join("upstream");
git::add_detached_worktree(&source.path, &worktree, &upstream_commit)?;
let scanned = catalog_inventory(&worktree, &source.selection);
let _ = git::remove_worktree(&source.path, &worktree);
let _ = git::prune_worktrees(&source.path);
scanned?
};
let outcomes = compare_catalog_inventory(&catalog_lock.inventory, &source.selection, &fresh);
Ok(CatalogDrift {
source_id: id.to_owned(),
pinned_commit: catalog_lock.commit,
upstream_commit,
outcomes,
migration_warnings,
})
}
pub fn advance_catalog(
paths: &StorePaths,
id: &str,
dry_run: bool,
) -> DaloResult<CatalogAdvanceReport> {
if dry_run {
ensure_no_pending_catalog_advance(paths)?;
} else {
recover_pending_catalog_advance(paths)?;
}
let config = store::read_config(paths)?;
let source = catalog_source_from_config(&config.sources, id)?;
if let Some(team) = &source.declared_by {
return Err(DaloError::StateError {
reason: format!(
"catalog `{id}` is pinned by `{team}`; update its version in `{}`",
crate::team_manifest::TEAM_MANIFEST_FILE
),
});
}
ensure_catalog_checkout(id, &source.path)?;
if git::is_dirty(&source.path)? {
return Err(DaloError::DirtySource {
source_id: id.to_owned(),
path: source.path,
});
}
let persisted_source_lock = read_source_lock(paths)?;
let mut source_lock = persisted_source_lock.clone();
let migration_warnings =
migrate_source_lock_inventories(&mut source_lock, &config.sources, Some(id))?;
let old_lock = source_lock
.catalog(id)
.ok_or_else(|| DaloError::unknown_source(id, Vec::new()))?
.clone();
ensure_catalog_checkout_matches_pin(id, &source.path, &old_lock)?;
git::fetch(&source.path)?;
git::prune_worktrees(&source.path)?;
let upstream_commit = git::rev_parse(&source.path, "FETCH_HEAD")?;
if git::revision_count(&source.path, &upstream_commit, &old_lock.commit)? != 0 {
return Err(DaloError::CatalogNotFastForward {
catalog_id: id.to_string(),
upstream: upstream_commit,
pinned: old_lock.commit,
});
}
let temp = dry_run.then(tempfile::tempdir).transpose()?;
let worktree = if let Some(temp) = &temp {
temp.path().join("upstream")
} else {
let staging_root = paths.sources_dir.join(".audit-staging");
fs::create_dir_all(&staging_root)?;
let staging_path = staging_root.join(format!("{id}-{upstream_commit}"));
cleanup_obsolete_catalog_staging(&source, &staging_root, &staging_path)?;
staging_path
};
let staging_matches = worktree.exists()
&& git::rev_parse_head(&worktree).is_ok_and(|commit| commit == upstream_commit);
if !staging_matches {
if worktree.exists() {
let _ = git::remove_worktree(&source.path, &worktree);
let _ = fs::remove_dir_all(&worktree);
git::prune_worktrees(&source.path)?;
}
git::add_detached_worktree(&source.path, &worktree, &upstream_commit)?;
}
let result = advance_catalog_candidate(
paths,
id,
dry_run,
&source,
config,
persisted_source_lock,
source_lock,
old_lock,
&worktree,
upstream_commit,
migration_warnings,
);
let keep_for_audit = !dry_run
&& result.as_ref().is_ok_and(|report| {
report
.blocking_reasons
.iter()
.any(|reason| reason.starts_with("security audit blocks"))
});
if !keep_for_audit {
let _ = git::remove_worktree(&source.path, &worktree);
let _ = git::prune_worktrees(&source.path);
if !dry_run {
let _ = fs::remove_dir(paths.sources_dir.join(".audit-staging"));
}
}
result
}
fn ensure_catalog_checkout(id: &str, path: &Path) -> DaloResult<()> {
let unavailable = |error: Option<&std::io::Error>| DaloError::StateError {
reason: match error {
Some(error) => format!(
"catalog `{id}` checkout at `{}` is unavailable ({error}); restore it or remove and re-add the catalog",
path.display()
),
None => format!(
"catalog `{id}` checkout at `{}` is unavailable; restore it or remove and re-add the catalog",
path.display()
),
},
};
let metadata = fs::metadata(path).map_err(|error| unavailable(Some(&error)))?;
if !metadata.is_dir() {
return Err(unavailable(None));
}
match path.join(".git").try_exists() {
Ok(true) => Ok(()),
Ok(false) => Err(unavailable(None)),
Err(error) => Err(unavailable(Some(&error))),
}
}
fn ensure_catalog_checkout_matches_pin(
id: &str,
path: &Path,
catalog_lock: &CatalogLock,
) -> DaloResult<()> {
let checkout_commit = git::rev_parse_head(path)?;
if checkout_commit != catalog_lock.commit {
return Err(DaloError::StateError {
reason: format!(
"catalog `{id}` checkout is at {checkout_commit}, but its pin is {}; restore the pinned checkout before advancing",
catalog_lock.commit
),
});
}
Ok(())
}
fn cleanup_obsolete_catalog_staging(
source: &SourceConfig,
staging_root: &Path,
keep: &Path,
) -> DaloResult<()> {
for entry in fs::read_dir(staging_root)? {
let entry = entry?;
let path = entry.path();
if path == keep || !is_catalog_staging_entry(&entry.file_name(), &source.id) {
continue;
}
if git::remove_worktree(&source.path, &path).is_err() {
fs::remove_dir_all(&path)?;
}
}
git::prune_worktrees(&source.path)
}
fn is_catalog_staging_entry(file_name: &std::ffi::OsStr, source_id: &str) -> bool {
let Some(file_name) = file_name.to_str() else {
return false;
};
let Some(commit) = file_name
.strip_prefix(source_id)
.and_then(|suffix| suffix.strip_prefix('-'))
else {
return false;
};
matches!(commit.len(), 40 | 64) && commit.bytes().all(|byte| byte.is_ascii_hexdigit())
}
#[allow(clippy::too_many_arguments)]
fn advance_catalog_candidate(
paths: &StorePaths,
id: &str,
dry_run: bool,
source: &SourceConfig,
original_config: UserConfig,
persisted_source_lock: SourceLock,
original_source_lock: SourceLock,
old_lock: CatalogLock,
worktree: &Path,
upstream_commit: String,
migration_warnings: Vec<String>,
) -> DaloResult<CatalogAdvanceReport> {
let fresh_for_comparison = catalog_inventory(worktree, &source.selection)?;
let outcomes = compare_catalog_inventory(
&old_lock.inventory,
&source.selection,
&fresh_for_comparison,
);
let selection_after = reconciled_selection(
&source.selection,
&old_lock.inventory,
&fresh_for_comparison,
);
let fresh_inventory = catalog_inventory(worktree, &selection_after)?;
let new_lock = CatalogLock {
source_id: id.to_owned(),
commit: upstream_commit.clone(),
selected: selection_after.clone(),
inventory: fresh_inventory,
};
let mut next_config = original_config.clone();
let configured = next_config
.sources
.iter_mut()
.find(|candidate| candidate.id == id)
.expect("catalog source was resolved from this config");
configured.path = worktree.to_path_buf();
configured.selection = selection_after.clone();
let approvals = store::read_approvals(paths)?;
let live = resolver::resolve_from_config(&next_config, approvals.approvals);
let mut blocking_reasons = Vec::new();
if let Some(scan) = live.scans.iter().find(|scan| scan.source.id == id)
&& let Some(error) = &scan.error
{
blocking_reasons.push(format!(
"candidate catalog inventory could not be scanned: {error}"
));
}
for outcome in &outcomes {
if outcome.code == DriftCode::SelectedRemoved {
blocking_reasons.push(format!(
"selected skill `{}` was removed; update the selection explicitly before advancing",
outcome.skill
));
}
}
let candidate_skills = candidate_resolution_skills(&live.resolution, id);
let audits = candidate_skills
.values()
.map(|skill| {
audit::audit_skill(
paths,
&skill.source_ref,
&skill.path,
&AuditOptions {
persist: false,
exclude_root_source_metadata: store::comparable_path(&skill.path)
== store::comparable_path(worktree),
..AuditOptions::default()
},
)
})
.collect::<DaloResult<Vec<_>>>()?;
for audit in &audits {
if audit.is_blocking() {
blocking_reasons.push(format!(
"security audit blocks candidate skill `{}`; review with `dalo audit '{}' --accept-risk <reason>`",
audit.source_ref,
audit.skill_path.display()
));
}
}
let candidate_plugins = live.plugins;
let mut candidate_resolution = live.resolution;
rebase_resolution_paths(&mut candidate_resolution, worktree, &source.path);
let preview = materialize::materialize(paths, &candidate_resolution, true)?;
let state = store::read_state(paths)?;
for operation in &preview.operations {
let changes_state = !matches!(
operation.kind,
MaterializeOperationKind::NoOp | MaterializeOperationKind::Keep
);
if changes_state && !operation_is_catalog_related(operation, &state, &source.path) {
blocking_reasons.push(
"materialization drift outside this catalog must be reconciled with `dalo sync` first"
.to_owned(),
);
break;
}
if operation.status == materialize::MaterializeOperationStatus::Blocked
&& operation_is_catalog_related(operation, &state, &source.path)
{
blocking_reasons.push(format!(
"catalog materialization is blocked at `{}`: {}",
operation.link_path.display(),
operation.reason.as_deref().unwrap_or("unsafe target state")
));
}
}
blocking_reasons.sort();
blocking_reasons.dedup();
let same_revision = old_lock.commit == upstream_commit;
if same_revision {
blocking_reasons.clear();
}
let mut report = CatalogAdvanceReport {
source_id: id.to_owned(),
old_lock,
new_lock,
outcomes,
selection_before: source.selection.clone(),
selection_after,
audits,
sync: preview,
blocking_reasons,
migration_warnings,
dry_run,
advanced: false,
};
if dry_run || same_revision || !report.blocking_reasons.is_empty() {
return Ok(report);
}
let original_user_lock = store::read_user_lock(paths)?;
let mut next_source_lock = original_source_lock;
next_source_lock
.catalogs
.retain(|catalog| catalog.source_id != id);
next_source_lock.catalogs.push(report.new_lock.clone());
next_source_lock
.catalogs
.sort_by(|left, right| left.source_id.cmp(&right.source_id));
let mut persisted_config = original_config.clone();
persisted_config
.sources
.iter_mut()
.find(|candidate| candidate.id == id)
.expect("catalog source was resolved from this config")
.selection = report.selection_after.clone();
write_catalog_advance_journal(
paths,
&CatalogAdvanceJournal {
schema_version: CATALOG_ADVANCE_SCHEMA_VERSION,
source_id: id.to_owned(),
original_commit: report.old_lock.commit.clone(),
original_config: original_config.clone(),
original_source_lock: persisted_source_lock.clone(),
original_user_lock: original_user_lock.clone(),
completed: false,
},
)?;
git::fast_forward_to(&source.path, &upstream_commit)?;
catalog_advance_hard_interrupt("checkout");
if let Err(error) = catalog_advance_boundary("checkout") {
return rollback_catalog_advance(
paths,
source,
&report.old_lock.commit,
&original_config,
&persisted_source_lock,
&original_user_lock,
None,
error,
);
}
let (applied, rollback) = match materialize::materialize_with_degraded_sources_rollback(
paths,
&candidate_resolution,
false,
&[],
) {
Ok(result) => result,
Err(error) => {
return rollback_catalog_advance(
paths,
source,
&report.old_lock.commit,
&original_config,
&persisted_source_lock,
&original_user_lock,
None,
error,
);
}
};
catalog_advance_hard_interrupt("materialization");
if let Err(error) = catalog_advance_boundary("materialization") {
return rollback_catalog_advance(
paths,
source,
&report.old_lock.commit,
&original_config,
&persisted_source_lock,
&original_user_lock,
rollback,
error,
);
}
if let Err(error) =
crate::delivery::verify_generated_source_snapshots(paths, &applied.resolution)
{
return rollback_catalog_advance(
paths,
source,
&report.old_lock.commit,
&original_config,
&persisted_source_lock,
&original_user_lock,
rollback,
error,
);
}
let mut next_user_lock = lockfile::build_user_lock(
&persisted_config.sources,
&applied.resolution,
Some(&applied),
Some(&candidate_plugins),
);
next_user_lock.active_instruction_packs = original_user_lock.active_instruction_packs.clone();
let persist = (|| -> DaloResult<()> {
write_source_lock(paths, &next_source_lock)?;
catalog_advance_hard_interrupt("source_lock");
catalog_advance_boundary("source_lock")?;
store::write_config(paths, &persisted_config)?;
catalog_advance_hard_interrupt("config");
catalog_advance_boundary("config")?;
store::write_user_lock(paths, &next_user_lock)?;
catalog_advance_hard_interrupt("user_lock");
catalog_advance_boundary("user_lock")
})();
if let Err(error) = persist {
return rollback_catalog_advance(
paths,
source,
&report.old_lock.commit,
&original_config,
&persisted_source_lock,
&original_user_lock,
rollback,
error,
);
}
if let Err(error) = mark_catalog_advance_complete(paths) {
return rollback_catalog_advance(
paths,
source,
&report.old_lock.commit,
&original_config,
&persisted_source_lock,
&original_user_lock,
rollback,
error,
);
}
let _ = clear_catalog_advance_journal(paths);
report.sync = applied;
report.advanced = true;
Ok(report)
}
fn reconciled_selection(
selection: &[String],
locked: &[CatalogEntry],
fresh: &[CatalogEntry],
) -> Vec<String> {
let mut seen = std::collections::BTreeSet::new();
let mut reconciled = Vec::with_capacity(selection.len());
for reference in selection {
let old = locked
.iter()
.find(|entry| entry_matches_ref(entry, reference));
let resolved = if let Some(old) = old
&& let Some(id) = old.id.as_deref()
&& unique_entry_with_id(locked, id).is_some()
&& let Some(candidate) = unique_entry_with_id(fresh, id)
&& candidate.path != old.path
{
id.to_owned()
} else {
reference.clone()
};
if seen.insert(resolved.clone()) {
reconciled.push(resolved);
}
}
reconciled
}
fn entry_matches_ref(entry: &CatalogEntry, reference: &str) -> bool {
entry.slot_name == reference
|| entry.path == reference
|| entry.id.as_deref() == Some(reference)
}
fn candidate_resolution_skills<'a>(
resolution: &'a Resolution,
source_id: &str,
) -> BTreeMap<String, &'a ResolvedSkill> {
let mut skills = BTreeMap::new();
for skill in resolution
.active_skills
.iter()
.chain(&resolution.pending_approval_skills)
.filter(|skill| skill.source_id == source_id)
{
skills.insert(skill.source_ref.clone(), skill);
}
for blocked in resolution
.blocked_skills
.iter()
.filter(|blocked| blocked.skill.source_id == source_id)
{
skills.insert(blocked.skill.source_ref.clone(), &blocked.skill);
}
skills
}
fn rebase_resolution_paths(resolution: &mut Resolution, from: &Path, to: &Path) {
let rebase = |skill: &mut ResolvedSkill| {
if let Ok(relative) = skill.path.strip_prefix(from) {
skill.path = to.join(relative);
}
};
for skill in &mut resolution.active_skills {
rebase(skill);
}
for skill in &mut resolution.pending_approval_skills {
rebase(skill);
}
for skill in &mut resolution.unlinked_skills {
rebase(&mut skill.skill);
}
for skill in &mut resolution.blocked_skills {
rebase(&mut skill.skill);
}
}
fn operation_is_catalog_related(
operation: &materialize::MaterializeOperation,
state: &store::StateFile,
source_path: &Path,
) -> bool {
operation
.desired_path
.as_deref()
.is_some_and(|path| path.starts_with(source_path))
|| state.owned_skills.iter().any(|owned| {
owned.link_path == operation.link_path && owned.store_path.starts_with(source_path)
})
}
#[allow(clippy::too_many_arguments)]
fn rollback_catalog_advance(
paths: &StorePaths,
source: &SourceConfig,
old_commit: &str,
original_config: &UserConfig,
original_source_lock: &SourceLock,
original_user_lock: &UserLock,
rollback: Option<MaterializationRollback>,
error: DaloError,
) -> DaloResult<CatalogAdvanceReport> {
let mut rollback_errors = Vec::new();
if let Some(rollback) = rollback
&& let Err(restore_error) = rollback.restore(paths)
{
rollback_errors.push(format!("materialization: {restore_error}"));
}
if let Err(restore_error) = git::reset_hard_to(&source.path, old_commit) {
rollback_errors.push(format!("checkout: {restore_error}"));
}
if let Err(restore_error) = store::write_config(paths, original_config) {
rollback_errors.push(format!("config: {restore_error}"));
}
if let Err(restore_error) = write_source_lock(paths, original_source_lock) {
rollback_errors.push(format!("source lock: {restore_error}"));
}
if let Err(restore_error) = store::write_user_lock(paths, original_user_lock) {
rollback_errors.push(format!("user lock: {restore_error}"));
}
if rollback_errors.is_empty() {
let _ = clear_catalog_advance_journal(paths);
Err(error)
} else {
Err(DaloError::Io(std::io::Error::other(format!(
"{error}; additionally failed to roll back catalog advance: {}",
rollback_errors.join("; ")
))))
}
}
fn read_catalog_advance_journal(paths: &StorePaths) -> DaloResult<Option<CatalogAdvanceJournal>> {
if !paths.catalog_advance_file.exists() {
return Ok(None);
}
let content = fs::read_to_string(&paths.catalog_advance_file)?;
let journal = toml::from_str::<CatalogAdvanceJournal>(&content).map_err(|error| {
DaloError::FileParse {
path: paths.catalog_advance_file.clone(),
reason: error.to_string(),
}
})?;
if journal.schema_version != CATALOG_ADVANCE_SCHEMA_VERSION {
return Err(DaloError::UnsupportedSchema {
path: paths.catalog_advance_file.clone(),
version: journal.schema_version,
supported: CATALOG_ADVANCE_SCHEMA_VERSION,
});
}
Ok(Some(journal))
}
fn write_catalog_advance_journal(
paths: &StorePaths,
journal: &CatalogAdvanceJournal,
) -> DaloResult<()> {
store::write_toml_atomic(&paths.catalog_advance_file, journal)
}
fn mark_catalog_advance_complete(paths: &StorePaths) -> DaloResult<()> {
let Some(mut journal) = read_catalog_advance_journal(paths)? else {
return Err(DaloError::StateError {
reason: "catalog advance recovery record disappeared before completion".to_owned(),
});
};
journal.completed = true;
write_catalog_advance_journal(paths, &journal)
}
fn clear_catalog_advance_journal(paths: &StorePaths) -> DaloResult<()> {
if paths.catalog_advance_file.exists() {
fs::remove_file(&paths.catalog_advance_file)?;
}
Ok(())
}
pub fn ensure_no_pending_catalog_advance(paths: &StorePaths) -> DaloResult<()> {
if let Some(journal) = read_catalog_advance_journal(paths)?
&& !journal.completed
{
return Err(DaloError::StateError {
reason: format!(
"catalog `{}` has an interrupted advance; run `dalo sync` or retry `dalo source refresh {} --advance` to recover it",
journal.source_id, journal.source_id
),
});
}
Ok(())
}
pub fn recover_pending_catalog_advance(paths: &StorePaths) -> DaloResult<()> {
let Some(journal) = read_catalog_advance_journal(paths)? else {
return Ok(());
};
if journal.completed {
return clear_catalog_advance_journal(paths);
}
let source = journal
.original_config
.sources
.iter()
.find(|source| source.id == journal.source_id)
.ok_or_else(|| DaloError::StateError {
reason: format!(
"catalog advance recovery for `{}` cannot find the original source configuration",
journal.source_id
),
})?;
ensure_catalog_checkout(&journal.source_id, &source.path)?;
if git::is_dirty(&source.path)? {
return Err(DaloError::StateError {
reason: format!(
"catalog `{}` has an interrupted advance and a dirty checkout at `{}`; restore or commit the checkout before recovery",
journal.source_id,
source.path.display()
),
});
}
git::reset_hard_to(&source.path, &journal.original_commit)?;
let approvals = store::read_approvals(paths)?;
let live = resolver::resolve_from_config(&journal.original_config, approvals.approvals);
materialize::materialize(paths, &live.resolution, false)?;
store::write_config(paths, &journal.original_config)?;
write_source_lock(paths, &journal.original_source_lock)?;
store::write_user_lock(paths, &journal.original_user_lock)?;
clear_catalog_advance_journal(paths)
}
fn catalog_advance_boundary(boundary: &str) -> DaloResult<()> {
if std::env::var("DALO_CATALOG_ADVANCE_FAIL_AT")
.ok()
.as_deref()
== Some(boundary)
{
return Err(DaloError::StateError {
reason: format!("injected catalog-advance failure at {boundary}"),
});
}
Ok(())
}
fn catalog_advance_hard_interrupt(boundary: &str) {
if std::env::var("DALO_CATALOG_ADVANCE_ABORT_AT")
.ok()
.as_deref()
== Some(boundary)
{
std::process::abort();
}
}
fn catalog_inventory_at_commit(
source_path: &Path,
commit: &str,
selection: &[String],
) -> DaloResult<Vec<CatalogEntry>> {
let temp = tempfile::tempdir()?;
let worktree = temp.path().join("pinned");
git::add_detached_worktree(source_path, &worktree, commit)?;
let scanned = catalog_inventory(&worktree, selection);
let _ = git::remove_worktree(source_path, &worktree);
let _ = git::prune_worktrees(source_path);
scanned
}
fn migrate_source_lock_inventories(
lock: &mut SourceLock,
sources: &[SourceConfig],
required_source_id: Option<&str>,
) -> DaloResult<Vec<String>> {
if lock.schema_version == SOURCE_LOCK_SCHEMA_VERSION {
return Ok(Vec::new());
}
let mut complete = true;
let mut warnings = Vec::new();
let mut migrated = Vec::with_capacity(lock.catalogs.len());
for mut catalog in std::mem::take(&mut lock.catalogs) {
let Some(source) = sources.iter().find(|source| source.id == catalog.source_id) else {
warnings.push(format!(
"dropped legacy lock entry for unconfigured catalog `{}`",
catalog.source_id
));
continue;
};
if source.kind != SourceKind::Catalog {
warnings.push(format!(
"dropped legacy lock entry `{}` because it is no longer a catalog source",
source.id
));
continue;
}
match catalog_inventory_at_commit(&source.path, &catalog.commit, &source.selection) {
Ok(inventory) => catalog.inventory = inventory,
Err(error) if required_source_id == Some(source.id.as_str()) => {
return Err(DaloError::CatalogMigrationFailed {
source_id: source.id.clone(),
reason: error.to_string(),
});
}
Err(error) => {
complete = false;
warnings.push(format!(
"skipped legacy inventory migration for catalog `{}`: {error}",
source.id
));
}
}
migrated.push(catalog);
}
lock.catalogs = migrated;
if complete {
lock.schema_version = SOURCE_LOCK_SCHEMA_VERSION;
}
Ok(warnings)
}
fn same_entry(a: &CatalogEntry, b: &CatalogEntry) -> bool {
match (a.id.as_deref(), b.id.as_deref()) {
(Some(left), Some(right)) => left == right,
_ => a.path == b.path,
}
}
fn unique_entry_with_id<'a>(entries: &'a [CatalogEntry], id: &str) -> Option<&'a CatalogEntry> {
let mut matches = entries
.iter()
.filter(|entry| entry.id.as_deref() == Some(id));
let first = matches.next()?;
matches.next().is_none().then_some(first)
}
fn relative_path(root: &Path, path: &Path) -> String {
path.strip_prefix(root)
.unwrap_or(path)
.to_string_lossy()
.into_owned()
}
pub fn hash_directory(skill_dir: &Path) -> DaloResult<String> {
hash_directory_with_options(skill_dir, false)
}
pub(crate) fn hash_source_root_directory(skill_dir: &Path) -> DaloResult<String> {
hash_directory_with_options(skill_dir, true)
}
fn hash_skill_directory(source_root: &Path, skill_dir: &Path) -> DaloResult<String> {
hash_directory_with_options(
skill_dir,
store::comparable_path(source_root) == store::comparable_path(skill_dir),
)
}
fn hash_directory_with_options(
skill_dir: &Path,
exclude_root_git_metadata: bool,
) -> DaloResult<String> {
let mut files = Vec::new();
collect_files(skill_dir, skill_dir, exclude_root_git_metadata, &mut files)?;
files.sort();
let mut hasher = Sha256::new();
for file in &files {
let relative = file.strip_prefix(skill_dir).unwrap_or(file.as_path());
hash_framed(&mut hasher, relative.as_os_str().as_bytes());
let metadata = fs::symlink_metadata(file)?;
if metadata.file_type().is_symlink() {
hasher.update(*b"L");
let target = fs::read_link(file)?;
hash_framed(&mut hasher, target.as_os_str().as_bytes());
} else if metadata.is_file() {
hasher.update(*b"F");
hasher.update((metadata.permissions().mode() & 0o111).to_le_bytes());
hasher.update(metadata.len().to_le_bytes());
let mut handle = fs::File::open(file)?;
let mut buffer = [0u8; 8192];
loop {
let read = handle.read(&mut buffer)?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
} else {
hasher.update(*b"S");
hasher.update(metadata.permissions().mode().to_le_bytes());
hasher.update(metadata.len().to_le_bytes());
}
}
Ok(hex_digest(&hasher.finalize()))
}
fn hash_framed(hasher: &mut Sha256, bytes: &[u8]) {
hasher.update((bytes.len() as u64).to_le_bytes());
hasher.update(bytes);
}
fn hash_metadata(skill: &SkillRecord) -> String {
let mut hasher = Sha256::new();
hasher.update(skill.id.clone().unwrap_or_default().as_bytes());
hasher.update([0]);
hasher.update(skill.description.clone().unwrap_or_default().as_bytes());
hasher.update([0]);
for requirement in &skill.requires {
hasher.update(requirement.as_bytes());
hasher.update([0]);
}
hasher.update([1]);
for tag in &skill.tags {
hasher.update(tag.as_bytes());
hasher.update([0]);
}
hex_digest(&hasher.finalize())
}
fn collect_files(
root: &Path,
dir: &Path,
exclude_root_git_metadata: bool,
files: &mut Vec<std::path::PathBuf>,
) -> DaloResult<()> {
for entry in fs::read_dir(dir)? {
let entry = entry?;
if exclude_root_git_metadata && dir == root && entry.file_name() == ".git" {
continue;
}
let file_type = entry.file_type()?;
let path = entry.path();
if file_type.is_dir() {
collect_files(root, &path, exclude_root_git_metadata, files)?;
} else {
files.push(path);
}
}
Ok(())
}
fn hex_digest(bytes: &[u8]) -> String {
use std::fmt::Write as _;
let mut output = String::with_capacity(bytes.len() * 2);
for byte in bytes {
let _ = write!(&mut output, "{byte:02x}");
}
output
}
#[cfg(test)]
mod tests {
use super::*;
use crate::inventory::{InventoryWarning, InventoryWarningCode};
use std::os::unix::fs::symlink;
fn entry(id: Option<&str>, slot: &str, path: &str, content: &str) -> CatalogEntry {
CatalogEntry {
id: id.map(str::to_owned),
slot_name: slot.to_owned(),
path: path.to_owned(),
content_hash: content.to_owned(),
metadata_hash: "m".to_owned(),
requires: Vec::new(),
}
}
fn candidate(id: Option<&str>, slot: &str, path: &str) -> CatalogCandidate {
CatalogCandidate {
id: id.map(str::to_owned),
slot_name: slot.to_owned(),
path: path.to_owned(),
description: None,
compatibility: None,
requires: Vec::new(),
selected: false,
}
}
#[test]
fn catalog_inspect_rejects_a_traversal_truncated_inventory() {
let mut inventory = SourceInventory {
source_id: "marketing".to_owned(),
skills: Vec::new(),
agents: Vec::new(),
plugins: Vec::new(),
warnings: vec![InventoryWarning {
code: InventoryWarningCode::UnreadablePath,
path: PathBuf::from("."),
message: "source inventory is incomplete: the 250000-entry source scan limit was reached at `later`; skills later in this source may be omitted. Reduce or split the source tree, then run `dalo sync`".to_owned(),
}],
agent_warnings: Vec::new(),
plugin_warnings: Vec::new(),
};
let error = ensure_catalog_inventory_complete(&inventory)
.expect_err("incomplete scan must not be presented as a full candidate list");
assert!(matches!(error, DaloError::StateError { .. }));
for detail in [
"catalog inventory is incomplete",
"source scan safety limit was reached",
"skills later in the tree may be missing",
"Reduce or split the source tree",
] {
assert!(
error.to_string().contains(detail),
"inspection error {error:?} lacks {detail:?}"
);
}
assert!(!error.to_string().contains("`later`"));
inventory.warnings[0].code = InventoryWarningCode::SkippedSymlink;
assert!(ensure_catalog_inventory_complete(&inventory).is_ok());
inventory.warnings[0].code = InventoryWarningCode::UnreadablePath;
inventory.warnings[0].message = "a skill path could not be read".to_owned();
assert!(ensure_catalog_inventory_complete(&inventory).is_ok());
}
#[test]
fn source_qualified_reference_preserves_exact_colon_bearing_stable_id() {
let candidates = vec![candidate(
Some("marketing:copy-editing"),
"renamed-copy-editing",
"skills/renamed-copy-editing",
)];
let resolved =
resolve_candidate_reference("marketing", &candidates, "marketing:copy-editing")
.expect("exact stable ID should resolve");
assert_eq!(resolved.slot_name, "renamed-copy-editing");
}
#[test]
fn source_qualified_reference_reports_exact_and_slot_collision() {
let candidates = vec![
candidate(
Some("marketing:copy-editing"),
"renamed-copy-editing",
"skills/renamed-copy-editing",
),
candidate(None, "copy-editing", "skills/copy-editing"),
];
let error = resolve_candidate_reference("marketing", &candidates, "marketing:copy-editing")
.expect_err("colliding exact and qualified references should be ambiguous");
assert!(matches!(error, DaloError::AmbiguousSkillReference { .. }));
assert!(error.to_string().contains("skills/renamed-copy-editing"));
assert!(error.to_string().contains("skills/copy-editing"));
}
#[test]
fn compare_should_flag_new_available_for_unselected_additions() {
let locked = vec![entry(Some("a"), "alpha", "skills/alpha", "h1")];
let fresh = vec![
entry(Some("a"), "alpha", "skills/alpha", "h1"),
entry(Some("b"), "beta", "skills/beta", "h2"),
];
let outcomes = compare_catalog_inventory(&locked, &["alpha".to_owned()], &fresh);
assert_eq!(outcomes.len(), 1);
assert_eq!(outcomes[0].code, DriftCode::NewAvailable);
assert_eq!(outcomes[0].skill, "beta");
}
#[test]
fn compare_should_flag_selected_changed_on_content_hash() {
let locked = vec![entry(Some("a"), "alpha", "skills/alpha", "h1")];
let fresh = vec![entry(Some("a"), "alpha", "skills/alpha", "h2")];
let outcomes = compare_catalog_inventory(&locked, &["alpha".to_owned()], &fresh);
assert_eq!(outcomes.len(), 1);
assert_eq!(outcomes[0].code, DriftCode::SelectedChanged);
}
#[test]
fn compare_should_flag_selected_moved_via_stable_id() {
let locked = vec![entry(Some("a"), "alpha", "skills/alpha", "h1")];
let fresh = vec![entry(Some("a"), "alpha", "catalog/alpha", "h1")];
let outcomes = compare_catalog_inventory(&locked, &["alpha".to_owned()], &fresh);
assert_eq!(outcomes.len(), 1);
assert_eq!(outcomes[0].code, DriftCode::SelectedMoved);
}
#[test]
fn compare_should_flag_selected_moved_and_changed_via_stable_id() {
let locked = vec![entry(Some("a"), "alpha", "skills/alpha", "h1")];
let fresh = vec![entry(Some("a"), "alpha", "catalog/alpha", "h2")];
let outcomes = compare_catalog_inventory(&locked, &["alpha".to_owned()], &fresh);
assert_eq!(outcomes.len(), 2);
assert_eq!(outcomes[0].code, DriftCode::SelectedChanged);
assert_eq!(outcomes[1].code, DriftCode::SelectedMoved);
assert_eq!(outcomes[0].skill, "a");
assert_eq!(outcomes[1].skill, "a");
}
#[test]
fn compare_should_flag_selected_removed_when_absent() {
let locked = vec![entry(Some("a"), "alpha", "skills/alpha", "h1")];
let fresh: Vec<CatalogEntry> = Vec::new();
let outcomes = compare_catalog_inventory(&locked, &["alpha".to_owned()], &fresh);
assert_eq!(outcomes.len(), 1);
assert_eq!(outcomes[0].code, DriftCode::SelectedRemoved);
assert!(outcomes[0].code.blocks_sync());
}
#[test]
fn compare_should_ignore_unselected_changes() {
let locked = vec![entry(Some("a"), "alpha", "skills/alpha", "h1")];
let fresh = vec![entry(Some("a"), "alpha", "skills/alpha", "h2")];
let outcomes = compare_catalog_inventory(&locked, &[], &fresh);
assert!(outcomes.is_empty());
}
#[test]
fn catalog_staging_entry_should_require_exact_source_id_and_commit_shape() {
let commit = "0123456789abcdef0123456789abcdef01234567";
assert!(is_catalog_staging_entry(
std::ffi::OsStr::new(&format!("foo-{commit}")),
"foo"
));
assert!(!is_catalog_staging_entry(
std::ffi::OsStr::new(&format!("foo-bar-{commit}")),
"foo"
));
assert!(!is_catalog_staging_entry(
std::ffi::OsStr::new("foo-not-a-commit"),
"foo"
));
}
#[test]
fn reconciled_selection_should_preserve_order_while_deduplicating_moves() {
let locked = vec![
entry(Some("a"), "alpha", "skills/alpha", "h1"),
entry(Some("b"), "beta", "skills/beta", "h2"),
];
let fresh = vec![
entry(Some("a"), "alpha", "catalog/alpha", "h1"),
entry(Some("b"), "beta", "skills/beta", "h2"),
];
let reconciled = reconciled_selection(
&["beta".to_owned(), "alpha".to_owned(), "a".to_owned()],
&locked,
&fresh,
);
assert_eq!(reconciled, vec!["beta", "a"]);
}
#[test]
fn hash_directory_should_include_symlink_targets() {
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
let skill_dir = temp_dir.path().join("skill");
fs::create_dir_all(&skill_dir).expect("skill dir should be created");
fs::write(skill_dir.join("SKILL.md"), "# Skill\n").expect("skill file should be written");
fs::write(skill_dir.join("a.md"), "A\n").expect("target a should be written");
fs::write(skill_dir.join("b.md"), "B\n").expect("target b should be written");
symlink("a.md", skill_dir.join("linked.md")).expect("symlink should be created");
let first = hash_directory(&skill_dir).expect("hash should succeed");
fs::remove_file(skill_dir.join("linked.md")).expect("symlink should be removed");
symlink("b.md", skill_dir.join("linked.md")).expect("symlink should be recreated");
let second = hash_directory(&skill_dir).expect("hash should succeed");
assert_ne!(first, second);
}
#[test]
fn hash_directory_should_include_executable_file_bits() {
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
let skill_dir = temp_dir.path().join("skill");
let script = skill_dir.join("script.sh");
fs::create_dir_all(&skill_dir).expect("skill dir should be created");
fs::write(&script, "#!/bin/sh\n").expect("script should be written");
let mut permissions = fs::metadata(&script)
.expect("script metadata should be readable")
.permissions();
permissions.set_mode(0o644);
fs::set_permissions(&script, permissions.clone()).expect("script should be non-executable");
let before = hash_directory(&skill_dir).expect("hash should succeed");
permissions.set_mode(0o744);
fs::set_permissions(&script, permissions).expect("script should be executable");
let after = hash_directory(&skill_dir).expect("hash should succeed");
assert_ne!(before, after);
}
#[test]
fn source_root_hash_should_exclude_checkout_git_metadata() {
let temp = tempfile::tempdir().expect("tempdir should be created");
let skill_dir = temp.path().join("skill");
fs::create_dir_all(skill_dir.join(".git"))
.expect("checkout metadata directory should be created");
fs::write(skill_dir.join("SKILL.md"), "# Root skill\n").expect("skill should be written");
fs::write(skill_dir.join(".git/HEAD"), "ref: refs/heads/main\n")
.expect("checkout metadata should be written");
let before =
hash_source_root_directory(&skill_dir).expect("source-root hash should succeed");
fs::write(skill_dir.join(".git/HEAD"), "ref: refs/heads/other\n")
.expect("checkout metadata should change");
let after =
hash_source_root_directory(&skill_dir).expect("source-root hash should succeed");
assert_eq!(before, after);
}
}