use crate::config::{Config, Settings};
use crate::dirs;
use crate::file::{self, display_path};
use crate::hash;
use crate::rand::random_string;
use crate::task::Task;
use eyre::{Result, bail};
use flate2::Compression;
use flate2::read::ZlibDecoder;
use flate2::write::ZlibEncoder;
use glob::glob;
use ignore::overrides::{Override, OverrideBuilder};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fs::{self, File};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use walkdir::WalkDir;
pub async fn remove_auto_output(task: &Task, config: &Arc<Config>) -> Result<()> {
if !task.outputs.is_auto() {
return Ok(());
}
let root = task_cwd(task, config).await?;
for output in task.outputs.paths(task, &root) {
match fs::remove_file(&output) {
Ok(()) => debug!("removed auto output file: {output}"),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err.into()),
}
}
Ok(())
}
pub fn is_glob_pattern(path: &str) -> bool {
let glob_chars = ['*', '{', '}'];
path.chars().any(|c| glob_chars.contains(&c))
}
const MAX_BRACE_EXPANSIONS: usize = 1024;
pub(crate) fn expand_glob_braces(pattern: &str) -> Result<Vec<String>> {
struct BraceGroup {
start: usize,
end: usize,
branches: Vec<(usize, usize)>,
}
fn find_group(pattern: &str) -> Result<Option<BraceGroup>> {
let mut escaped = false;
let mut class_depth = 0;
let mut group_start = None;
let mut group_depth = 0;
let mut branch_start = 0;
let mut branches = Vec::new();
for (idx, ch) in pattern.char_indices() {
if escaped {
escaped = false;
continue;
}
if ch == '\\' && !cfg!(windows) {
escaped = true;
continue;
}
match ch {
'[' if class_depth == 0 => class_depth = 1,
']' if class_depth == 1 => class_depth = 0,
'{' if class_depth == 0 => {
if group_depth == 0 {
group_start = Some(idx);
branch_start = idx + ch.len_utf8();
}
group_depth += 1;
}
',' if class_depth == 0 && group_depth == 1 => {
branches.push((branch_start, idx));
branch_start = idx + ch.len_utf8();
}
'}' if class_depth == 0 => {
if group_depth == 0 {
bail!("unopened brace alternate in glob pattern {pattern:?}");
}
group_depth -= 1;
if group_depth == 0 {
let start = group_start.unwrap();
if !branches.is_empty() {
branches.push((branch_start, idx));
return Ok(Some(BraceGroup {
start,
end: idx,
branches,
}));
}
if let Some(mut nested) = find_group(&pattern[start + 1..idx])? {
let offset = start + 1;
nested.start += offset;
nested.end += offset;
for (branch_start, branch_end) in &mut nested.branches {
*branch_start += offset;
*branch_end += offset;
}
return Ok(Some(nested));
}
group_start = None;
branches.clear();
}
}
_ => {}
}
}
if group_depth != 0 {
bail!("unclosed brace alternate in glob pattern {pattern:?}");
}
Ok(None)
}
fn expand(pattern: &str, expanded: &mut Vec<String>) -> Result<()> {
let Some(group) = find_group(pattern)? else {
if expanded.len() >= MAX_BRACE_EXPANSIONS {
bail!(
"glob pattern expands to more than {MAX_BRACE_EXPANSIONS} alternatives: {pattern:?}"
);
}
let mut literal = String::with_capacity(pattern.len());
let mut escaped = false;
let mut class_depth = 0;
for ch in pattern.chars() {
if escaped {
escaped = false;
literal.push(ch);
continue;
}
if ch == '\\' && !cfg!(windows) {
escaped = true;
literal.push(ch);
continue;
}
match ch {
'[' if class_depth == 0 => {
class_depth = 1;
literal.push(ch);
}
']' if class_depth == 1 => {
class_depth = 0;
literal.push(ch);
}
'{' if class_depth == 0 => literal.push_str("[{]"),
'}' if class_depth == 0 => literal.push_str("[}]"),
_ => literal.push(ch),
}
}
expanded.push(literal);
return Ok(());
};
let prefix = &pattern[..group.start];
let suffix = &pattern[group.end + 1..];
for (branch_start, branch_end) in group.branches {
let branch = &pattern[branch_start..branch_end];
if branch.is_empty() {
continue;
}
expand(&format!("{prefix}{branch}{suffix}"), expanded)?;
}
Ok(())
}
let mut expanded = Vec::new();
expand(pattern, &mut expanded)?;
Ok(expanded)
}
pub(crate) fn build_source_matcher(
match_root: &Path,
task_cwd: &Path,
sources: &[String],
) -> Override {
let mut builder = OverrideBuilder::new(match_root);
for s in sources {
let normalized = normalize_pattern(match_root, task_cwd, s);
let expanded = match expand_glob_braces(&normalized) {
Ok(expanded) => expanded,
Err(e) => {
warn!("invalid source pattern {s:?}: {e}");
continue;
}
};
for normalized in expanded {
if let Err(e) = builder.add(&normalized) {
warn!("invalid source pattern {s:?}: {e}");
}
}
}
builder.build().unwrap_or_else(|e| {
warn!("failed to build source matcher: {e}");
Override::empty()
})
}
fn normalize_pattern(match_root: &Path, task_cwd: &Path, pattern: &str) -> String {
let (prefix, body) = if pattern.starts_with("\\!") {
return pattern.to_string();
} else if let Some(rest) = pattern.strip_prefix('!') {
("!", rest)
} else {
("", pattern)
};
let body_path = Path::new(body);
if body_path.is_absolute() {
if let Ok(rel) = body_path.strip_prefix(match_root)
&& let Some(rel_str) = rel.to_str()
{
let rel_str = if rel_str.starts_with('!') {
format!("\\{rel_str}")
} else {
rel_str.to_string()
};
return format!("{prefix}{rel_str}");
}
return pattern.to_string();
}
if let Ok(cwd_rel) = task_cwd.strip_prefix(match_root)
&& let Some(cwd_rel_str) = cwd_rel.to_str()
&& !cwd_rel_str.is_empty()
{
return format!("{prefix}{cwd_rel_str}/{body}");
}
pattern.to_string()
}
pub(crate) fn is_source(matcher: &Override, path: &Path) -> bool {
if path.is_absolute() && !path.starts_with(matcher.path()) {
return true;
}
matcher.matched(path, false).is_whitelist()
}
pub(crate) fn source_glob_patterns(sources: &[String]) -> Vec<String> {
sources
.iter()
.filter_map(|s| {
if s.starts_with('!') {
None
} else if let Some(rest) = s.strip_prefix("\\!") {
Some(format!("!{rest}"))
} else {
Some(s.clone())
}
})
.collect()
}
pub(crate) fn build_output_matcher(root: &Path, outputs: &[String]) -> Result<Override> {
let mut builder = OverrideBuilder::new(root);
for output in outputs {
let output = normalize_pattern(root, root, output);
for output in expand_glob_braces(&output)? {
builder.add(&output)?;
let descendant = if let Some(body) = output.strip_prefix('!') {
format!("!{body}/**")
} else if let Some(body) = output.strip_prefix("\\!") {
format!("\\!{body}/**")
} else {
format!("{output}/**")
};
if !output.ends_with("/**") {
builder.add(&descendant)?;
}
}
}
Ok(builder.build()?)
}
pub(crate) fn output_glob_patterns(outputs: &[String]) -> Vec<String> {
source_glob_patterns(outputs)
}
pub(crate) fn is_output(matcher: &Override, path: &Path, is_dir: bool) -> bool {
if path.is_absolute() && !path.starts_with(matcher.path()) {
return true;
}
matcher.matched(path, is_dir).is_whitelist()
}
fn resolve_task_path(root: &Path, path: impl AsRef<Path>) -> PathBuf {
let path = path.as_ref();
if path.is_absolute() {
path.to_path_buf()
} else {
root.join(path)
}
}
pub async fn task_cwd(task: &Task, config: &Arc<Config>) -> Result<PathBuf> {
if let Some(d) = task.dir(config).await? {
Ok(d)
} else {
Ok(config
.project_root
.clone()
.or_else(|| dirs::CWD.clone())
.unwrap_or_default())
}
}
async fn collect_source_metadatas(
task: &Task,
config: &Arc<Config>,
) -> Result<(PathBuf, PathBuf, Vec<(PathBuf, fs::Metadata)>)> {
let root = task_cwd(task, config).await?;
let match_root_owned = config
.config_files
.values()
.filter_map(|cf| cf.project_root())
.filter(|pr| root.starts_with(pr) || *pr == root)
.min_by_key(|p| p.components().count())
.unwrap_or_else(|| root.clone());
let match_root = match_root_owned.as_path();
let matcher = build_source_matcher(match_root, &root, &task.sources);
let glob_patterns = source_glob_patterns(&task.sources);
let mut source_metadatas = get_file_metadatas(&root, &glob_patterns, &matcher)?;
for config_source in task.config_sources() {
let config_path = if config_source.is_absolute() {
config_source.to_path_buf()
} else {
root.join(config_source)
};
if let Ok(meta) = config_path.metadata()
&& meta.is_file()
&& !source_metadatas.iter().any(|(p, _)| p == &config_path)
{
source_metadatas.push((config_path, meta));
}
}
Ok((root, match_root_owned, source_metadatas))
}
async fn compute_source_hash(
task: &Task,
config: &Arc<Config>,
) -> Result<Option<(String, PathBuf)>> {
if task.sources.is_empty() {
return Ok(None);
}
let use_content_hash = Settings::get().task.source_freshness_hash_contents;
let (root, _, source_metadatas) = collect_source_metadatas(task, config).await?;
if source_metadatas.is_empty() {
return Ok(None);
}
let source_hash = if use_content_hash {
let cache_path = content_hash_cache_path(task, &root);
let mut cache = load_content_hash_cache(&cache_path);
let h = file_contents_to_hash(&source_metadatas, &mut cache)?;
if let Err(e) = save_content_hash_cache(&cache_path, &cache) {
trace!("failed to save content hash cache: {e}");
}
h
} else {
file_metadatas_to_hash(&source_metadatas)
};
let source_hash_path = sources_hash_path(task, &root, use_content_hash);
Ok(Some((source_hash, source_hash_path)))
}
pub struct TaskCacheInputs {
pub source_hash: String,
pub source_paths: Vec<PathBuf>,
pub root_identity: PathBuf,
}
pub async fn task_cache_inputs(
task: &Task,
config: &Arc<Config>,
persist_content_hash_cache: bool,
) -> Result<Option<TaskCacheInputs>> {
if task.sources.is_empty() {
return Ok(None);
}
let (root, match_root, mut source_metadatas) = collect_source_metadatas(task, config).await?;
if source_metadatas.is_empty() {
return Ok(None);
}
source_metadatas.sort_by(|(a, _), (b, _)| a.cmp(b));
let cache_path = content_hash_cache_path(task, &root);
let mut cache = load_content_hash_cache(&cache_path);
let mut next = ContentHashCache::new();
let mut hasher = blake3::Hasher::new();
let mut source_paths = Vec::with_capacity(source_metadatas.len());
for (path, metadata) in source_metadatas {
let identity = match path.strip_prefix(&match_root) {
Ok(relative) => format!("workspace\0{}", relative.to_string_lossy()),
Err(_) => format!("external\0{}", path.to_string_lossy()),
};
hasher.update(&(identity.len() as u64).to_le_bytes());
hasher.update(identity.as_bytes());
let contents = match cache.get(&path) {
Some(entry) if cached_entry_matches(entry, &metadata) => entry.hash.clone(),
_ => hash::file_hash_blake3(&path, None)?,
};
hasher.update(contents.as_bytes());
next.insert(path.clone(), make_cache_entry(&metadata, contents));
source_paths.push(path.strip_prefix(&root).unwrap_or(&path).to_path_buf());
}
cache = next;
if persist_content_hash_cache && let Err(e) = save_content_hash_cache(&cache_path, &cache) {
trace!("failed to save content hash cache: {e}");
}
let root_identity = root
.strip_prefix(&match_root)
.unwrap_or(&root)
.to_path_buf();
Ok(Some(TaskCacheInputs {
source_hash: hasher.finalize().to_hex().to_string(),
source_paths,
root_identity,
}))
}
pub async fn sources_are_fresh(task: &Task, config: &Arc<Config>) -> Result<bool> {
if task.sources.is_empty() {
return Ok(false);
}
let settings = Settings::get();
let use_content_hash = settings.task.source_freshness_hash_contents;
let equal_mtime_is_fresh = settings.task.source_freshness_equal_mtime_is_fresh;
let run = async || -> Result<bool> {
let (root, _, source_metadatas) = collect_source_metadatas(task, config).await?;
if source_metadatas.is_empty() {
warn!(
"task {} has sources defined but no matching files found",
task.name
);
return Ok(false);
}
if !use_content_hash {
for (path, metadata) in &source_metadatas {
if let Ok(mtime) = metadata.modified()
&& mtime == UNIX_EPOCH
{
debug!(
"source file {} has epoch timestamp, treating as stale",
display_path(path)
);
return Ok(false);
}
}
}
let source_hash = if use_content_hash {
let cache_path = content_hash_cache_path(task, &root);
let mut cache = load_content_hash_cache(&cache_path);
let h = file_contents_to_hash(&source_metadatas, &mut cache)?;
if let Err(e) = save_content_hash_cache(&cache_path, &cache) {
trace!("failed to save content hash cache: {e}");
}
h
} else {
file_metadatas_to_hash(&source_metadatas)
};
let source_hash_path = sources_hash_path(task, &root, use_content_hash);
if let Some(dir) = source_hash_path.parent() {
file::create_dir_all(dir)?;
}
let existing_hash = source_existing_hash(task, &root, use_content_hash);
if existing_hash.as_deref().is_some_and(|h| h != source_hash) {
debug!(
"source {} hash mismatch in {}",
if use_content_hash {
"content"
} else {
"metadata"
},
source_hash_path.display()
);
return Ok(false);
}
if use_content_hash {
if existing_hash.is_none() {
debug!("no stored content hash in {}", source_hash_path.display());
return Ok(false);
}
let current_output_hash = compute_output_hash(task, &root)?;
let stored_output_hash = output_existing_hash(task, &root);
let fresh = current_output_hash.is_some()
&& current_output_hash.as_deref() == stored_output_hash.as_deref();
file::write(&source_hash_path, &source_hash)?;
return Ok(fresh);
}
let sources = get_last_modified_from_metadatas(&source_metadatas);
let outputs = get_last_modified(&root, &task.outputs.paths(task, &root))?;
trace!("sources: {sources:?}, outputs: {outputs:?}");
let fresh = match (sources, outputs) {
(Some(sources), Some(outputs)) => {
if equal_mtime_is_fresh {
sources <= outputs
} else {
sources < outputs
}
}
_ => false,
};
if fresh {
file::write(&source_hash_path, &source_hash)?;
}
Ok(fresh)
};
Ok(run().await.unwrap_or_else(|err| {
warn!("sources_are_fresh: {err:?}");
false
}))
}
pub async fn save_checksum(task: &Task, config: &Arc<Config>) -> Result<()> {
if task.sources.is_empty() {
return Ok(());
}
let root = task_cwd(task, config).await?;
if task.outputs.is_auto() {
for p in task.outputs.paths(task, &root) {
debug!("touching auto output file: {p}");
file::touch_file(&PathBuf::from(&p))?;
}
} else {
for output in output_glob_patterns(&task.outputs.paths(task, &root)) {
let output_exists = if is_glob_pattern(&output) {
expand_glob_braces(&output)
.map(|patterns| {
patterns.into_iter().any(|pattern| {
let pattern = resolve_task_path(&root, pattern);
glob(pattern.to_str().unwrap_or_default())
.map(|paths| paths.flatten().next().is_some())
.unwrap_or(false)
})
})
.unwrap_or(false)
} else {
let path = Path::new(&output);
let full_path = if path.is_relative() {
root.join(path)
} else {
path.to_path_buf()
};
full_path.exists()
};
if !output_exists {
warn!(
"task {} did not generate expected output: {}",
task.name, output
);
}
}
}
if let Some((hash, path)) = compute_source_hash(task, config).await? {
if let Some(dir) = path.parent() {
file::create_dir_all(dir)?;
}
file::write(&path, &hash)?;
}
if Settings::get().task.source_freshness_hash_contents {
let out_path = outputs_hash_path(task, &root);
match compute_output_hash(task, &root) {
Ok(Some(h)) => {
if let Some(dir) = out_path.parent() {
file::create_dir_all(dir)?;
}
file::write(&out_path, &h)?;
}
Ok(None) => {} Err(e) => {
let _ = std::fs::remove_file(&out_path);
warn!(
"task {} output hashing failed; next run will not be skipped: {e}",
task.name
);
}
}
}
Ok(())
}
fn task_state_key(task: &Task, root: &Path) -> String {
let mut hasher = DefaultHasher::new();
task.hash(&mut hasher);
task.config_sources().hash(&mut hasher);
root.hash(&mut hasher);
task.run.hash(&mut hasher);
task.sources.hash(&mut hasher);
task.outputs.patterns().hash(&mut hasher);
format!("{:x}", hasher.finish())
}
fn sources_hash_path(task: &Task, root: &Path, content_hash: bool) -> PathBuf {
let suffix = if content_hash { "-content" } else { "" };
dirs::STATE
.join("task-sources")
.join(format!("{}{suffix}", task_state_key(task, root)))
}
fn source_existing_hash(task: &Task, root: &Path, content_hash: bool) -> Option<String> {
let path = sources_hash_path(task, root, content_hash);
if path.exists() {
Some(file::read_to_string(&path).unwrap_or_default())
} else {
None
}
}
fn outputs_hash_path(task: &Task, root: &Path) -> PathBuf {
dirs::STATE
.join("task-sources")
.join(format!("{}-outputs", task_state_key(task, root)))
}
fn output_existing_hash(task: &Task, root: &Path) -> Option<String> {
let path = outputs_hash_path(task, root);
if path.exists() {
Some(file::read_to_string(&path).unwrap_or_default())
} else {
None
}
}
fn compute_output_hash(task: &Task, root: &Path) -> Result<Option<String>> {
let raw_patterns = task.outputs.paths(task, root);
let matcher = build_output_matcher(root, &raw_patterns)?;
let patterns_or_paths = output_glob_patterns(&raw_patterns);
if patterns_or_paths.is_empty() {
return Ok(None);
}
let (glob_pats, static_paths): (Vec<&String>, Vec<&String>) =
patterns_or_paths.iter().partition(|p| is_glob_pattern(p));
let mut entries: Vec<(PathBuf, String)> = Vec::new();
fn hash_file(path: &Path) -> Result<(PathBuf, String)> {
Ok((path.to_path_buf(), hash::file_hash_blake3(path, None)?))
}
fn push_dir_entries(
dir: &Path,
entries: &mut Vec<(PathBuf, String)>,
matcher: &Override,
) -> Result<bool> {
let mut found_any = false;
for entry in WalkDir::new(dir).follow_links(true).into_iter() {
let entry = entry?;
let path = entry.path();
if path == dir {
continue; }
if !is_output(matcher, path, entry.file_type().is_dir()) {
continue;
}
if entry.file_type().is_file() {
entries.push(hash_file(path)?);
found_any = true;
} else if entry.file_type().is_dir() {
entries.push((path.to_path_buf(), "dir".to_string()));
found_any = true;
}
}
Ok(found_any)
}
for path_str in static_paths {
let path = {
let p = Path::new(path_str.as_str());
if p.is_relative() {
root.join(p)
} else {
p.to_path_buf()
}
};
match path.metadata() {
Ok(m) if m.is_file() => {
if is_output(&matcher, &path, false) {
entries.push(hash_file(&path)?);
} else {
continue;
}
}
Ok(m) if m.is_dir() => {
if !push_dir_entries(&path, &mut entries, &matcher)?
&& is_output(&matcher, &path, true)
{
entries.push((path, "empty-dir".to_string()));
}
}
Ok(_) => {
if is_output(&matcher, &path, false) {
entries.push((path, "other".to_string()));
}
}
Err(_) => {
if is_output(&matcher, &path, false) || is_output(&matcher, &path, true) {
return Ok(None); }
}
}
}
for pattern_str in glob_pats {
let mut glob_matched = false;
for expanded in expand_glob_braces(pattern_str)? {
let full = resolve_task_path(root, expanded);
for entry in glob(full.to_str().unwrap_or_default())? {
let path = entry?;
glob_matched = true;
let metadata = match path.metadata() {
Ok(metadata) => metadata,
Err(_) => {
if !is_output(&matcher, &path, false) && !is_output(&matcher, &path, true) {
continue;
}
return Ok(None); }
};
if !is_output(&matcher, &path, metadata.is_dir()) {
continue;
}
match metadata {
m if m.is_file() => {
entries.push(hash_file(&path)?);
}
m if m.is_dir() => {
let found = push_dir_entries(&path, &mut entries, &matcher)?;
if !found {
entries.push((path, "empty-dir".to_string()));
}
}
_ => {
entries.push((path, "other".to_string()));
}
}
}
}
if !glob_matched {
return Ok(None);
}
}
entries.sort_by(|(a, _), (b, _)| a.cmp(b));
Ok(Some(hash::hash_to_str(&entries)))
}
fn get_file_metadatas(
root: &Path,
patterns_or_paths: &[String],
matcher: &Override,
) -> Result<Vec<(PathBuf, fs::Metadata)>> {
if patterns_or_paths.is_empty() {
return Ok(vec![]);
}
let (patterns, paths): (Vec<&String>, Vec<&String>) =
patterns_or_paths.iter().partition(|p| is_glob_pattern(p));
let mut metadatas = BTreeMap::new();
for pattern in patterns {
for expanded in expand_glob_braces(pattern)? {
let pattern = resolve_task_path(root, expanded);
let files = glob(pattern.to_str().unwrap())?;
for file in files.flatten() {
if let Ok(metadata) = file.metadata() {
metadatas.insert(file, metadata);
}
}
}
}
for path in paths {
let file = resolve_task_path(root, path);
if let Ok(metadata) = file.metadata() {
metadatas.insert(file, metadata);
}
}
let metadatas = metadatas
.into_iter()
.filter(|(_, m)| m.is_file())
.filter(|(p, _)| is_source(matcher, p))
.collect_vec();
Ok(metadatas)
}
fn file_metadatas_to_hash(metadatas: &[(PathBuf, fs::Metadata)]) -> String {
let stat_info: Vec<_> = metadatas
.iter()
.map(|(p, m)| (p, m.len(), m.modified().ok()))
.collect();
hash::hash_to_str(&stat_info)
}
#[derive(Debug, Serialize, Deserialize)]
struct CachedFileHash {
mtime_secs: i64,
mtime_nanos: u32,
size: u64,
hash: String,
}
type ContentHashCache = BTreeMap<PathBuf, CachedFileHash>;
fn content_hash_cache_path(task: &Task, root: &Path) -> PathBuf {
dirs::STATE
.join("task-sources")
.join(format!("{}-content-cache", task_state_key(task, root)))
}
fn load_content_hash_cache(path: &Path) -> ContentHashCache {
(|| -> Result<ContentHashCache> {
let mut zlib = ZlibDecoder::new(File::open(path)?);
let mut bytes = Vec::new();
zlib.read_to_end(&mut bytes)?;
Ok(rmp_serde::from_slice(&bytes)?)
})()
.unwrap_or_default()
}
fn save_content_hash_cache(path: &Path, cache: &ContentHashCache) -> Result<()> {
if let Some(parent) = path.parent() {
file::create_dir_all(parent)?;
}
let partial = path.with_extension(format!("part-{}", random_string(8)));
{
let mut zlib = ZlibEncoder::new(File::create(&partial)?, Compression::fast());
zlib.write_all(&rmp_serde::to_vec_named(cache)?)?;
zlib.finish()?;
}
file::rename(&partial, path)?;
Ok(())
}
fn cached_entry_matches(entry: &CachedFileHash, metadata: &fs::Metadata) -> bool {
let Ok(mtime) = metadata.modified() else {
return false;
};
let Ok(dur) = mtime.duration_since(UNIX_EPOCH) else {
return false;
};
entry.size == metadata.len()
&& entry.mtime_secs == dur.as_secs() as i64
&& entry.mtime_nanos == dur.subsec_nanos()
}
fn make_cache_entry(metadata: &fs::Metadata, hash: String) -> CachedFileHash {
let dur = metadata
.modified()
.ok()
.and_then(|m| m.duration_since(UNIX_EPOCH).ok());
CachedFileHash {
mtime_secs: dur.map(|d| d.as_secs() as i64).unwrap_or(0),
mtime_nanos: dur.map(|d| d.subsec_nanos()).unwrap_or(0),
size: metadata.len(),
hash,
}
}
fn file_contents_to_hash(
metadatas: &[(PathBuf, fs::Metadata)],
cache: &mut ContentHashCache,
) -> Result<String> {
let mut content_hashes: Vec<(&PathBuf, String)> = Vec::new();
let mut next: ContentHashCache = BTreeMap::new();
for (path, metadata) in metadatas {
let hash = match cache.get(path) {
Some(entry) if cached_entry_matches(entry, metadata) => entry.hash.clone(),
_ => hash::file_hash_blake3(path, None)?,
};
next.insert(path.clone(), make_cache_entry(metadata, hash.clone()));
content_hashes.push((path, hash));
}
*cache = next;
Ok(hash::hash_to_str(&content_hashes))
}
fn get_last_modified_from_metadatas(metadatas: &[(PathBuf, fs::Metadata)]) -> Option<SystemTime> {
metadatas.iter().flat_map(|(_, m)| m.modified()).max()
}
fn get_last_modified(root: &Path, patterns_or_paths: &[String]) -> Result<Option<SystemTime>> {
if patterns_or_paths.is_empty() {
return Ok(None);
}
let matcher = build_output_matcher(root, patterns_or_paths)?;
let mut file_modified = Vec::new();
let mut directory_modified = Vec::new();
for pattern in output_glob_patterns(patterns_or_paths) {
let candidates = if is_glob_pattern(&pattern) {
let mut candidates = Vec::new();
for expanded in expand_glob_braces(&pattern)? {
let expanded = resolve_task_path(root, expanded);
candidates.extend(
glob(expanded.to_str().unwrap_or_default())?.collect::<Result<Vec<_>, _>>()?,
);
}
candidates
} else {
vec![resolve_task_path(root, &pattern)]
};
for candidate in candidates {
if fs::symlink_metadata(&candidate).is_err() {
continue;
}
for entry in WalkDir::new(candidate).follow_links(true) {
let entry = entry?;
let metadata = entry.metadata()?;
if is_output(&matcher, entry.path(), metadata.is_dir()) {
if metadata.is_dir() {
directory_modified.push(metadata.modified()?);
} else {
file_modified.push(metadata.modified()?);
}
}
}
}
}
let last_mod = file_modified.into_iter().chain(directory_modified).max();
trace!(
"last_modified of {}: {last_mod:?}",
patterns_or_paths.iter().join(" ")
);
Ok(last_mod)
}
#[cfg(test)]
mod tests {
use super::*;
fn matches(sources: &[&str], path: &str) -> bool {
let sources: Vec<String> = sources.iter().map(|s| s.to_string()).collect();
let root = Path::new(".");
let matcher = build_source_matcher(root, root, &sources);
is_source(&matcher, Path::new(path))
}
#[test]
fn output_matcher_excludes_and_reincludes_descendants() {
let root = Path::new("/project");
let patterns = vec![
"dist".to_string(),
"!dist/**/*.map".to_string(),
"dist/keep.map".to_string(),
];
let matcher = build_output_matcher(root, &patterns).unwrap();
assert!(is_output(&matcher, &root.join("dist/app.js"), false));
assert!(!is_output(&matcher, &root.join("dist/app.map"), false));
assert!(is_output(&matcher, &root.join("dist/nested/app.js"), false));
assert!(is_output(&matcher, &root.join("dist/keep.map"), false));
}
#[test]
fn output_matcher_normalizes_absolute_patterns_under_root() {
let root = tempfile::tempdir().unwrap();
let output = root.path().join("dist/result.txt");
let patterns = vec![format!("{}/dist/**/*", root.path().display())];
let matcher = build_output_matcher(root.path(), &patterns).unwrap();
assert!(is_output(&matcher, &output, false));
}
#[test]
fn metadata_hash_notices_a_same_size_change_with_an_older_mtime() {
let root = tempfile::tempdir().unwrap();
let p = root.path().join("pin.txt");
fs::write(&p, "1.2.3").unwrap();
let before = file_metadatas_to_hash(&[(p.clone(), fs::metadata(&p).unwrap())]);
fs::write(&p, "1.2.4").unwrap();
let restored = filetime::FileTime::from_unix_time(1_000_000, 0);
filetime::set_file_times(&p, restored, restored).unwrap();
let after = file_metadatas_to_hash(&[(p.clone(), fs::metadata(&p).unwrap())]);
assert_eq!(
fs::metadata(&p).unwrap().len(),
5,
"the fixture only exercises the bug while both versions are the same size"
);
assert_ne!(before, after, "the mtime change should be part of the hash");
}
#[test]
fn metadata_hash_separates_pre_epoch_mtimes() {
let root = tempfile::tempdir().unwrap();
let p = root.path().join("ancient.txt");
fs::write(&p, "x").unwrap();
let stamp = |secs: i64| {
let t = filetime::FileTime::from_unix_time(secs, 0);
filetime::set_file_times(&p, t, t).ok()?;
let metadata = fs::metadata(&p).unwrap();
let mtime = metadata.modified().ok()?;
Some((mtime, file_metadatas_to_hash(&[(p.clone(), metadata)])))
};
let (Some((first_mtime, first)), Some((second_mtime, second))) =
(stamp(-2_000_000), stamp(-1_000_000))
else {
return;
};
if first_mtime == second_mtime {
return;
}
assert_ne!(
first, second,
"two different pre-epoch mtimes should hash differently"
);
}
#[test]
fn output_globs_ignore_excludes_and_unescape_literal_bangs() {
assert_eq!(
output_glob_patterns(&[
"dist".to_string(),
"!dist/**/*.map".to_string(),
"\\!important".to_string(),
]),
["dist", "!important"]
);
}
#[test]
fn glob_braces_expand_nested_and_multiple_alternates() {
assert_eq!(
expand_glob_braces("src/{a,{b,c}}/{one,two}.txt").unwrap(),
[
"src/a/one.txt",
"src/a/two.txt",
"src/b/one.txt",
"src/b/two.txt",
"src/c/one.txt",
"src/c/two.txt",
]
);
assert_eq!(expand_glob_braces("{,a}.txt").unwrap(), ["a.txt"]);
assert!(expand_glob_braces("src/{a,b.txt").is_err());
assert!(expand_glob_braces(&"{a,b}".repeat(11)).is_err());
}
#[test]
fn glob_braces_preserve_literal_singleton_groups() {
assert_eq!(
expand_glob_braces("{generated}.txt").unwrap(),
["[{]generated[}].txt"]
);
assert_eq!(
expand_glob_braces("{generated}/{a,b}.txt").unwrap(),
["[{]generated[}]/a.txt", "[{]generated[}]/b.txt"]
);
assert_eq!(
expand_glob_braces("{generated}.{txt,out}").unwrap(),
["[{]generated[}].txt", "[{]generated[}].out"]
);
assert_eq!(
expand_glob_braces("{prefix-{a,b}}.txt").unwrap(),
["[{]prefix-a[}].txt", "[{]prefix-b[}].txt"]
);
}
#[cfg(not(windows))]
#[test]
fn glob_braces_preserve_escaped_unix_braces() {
assert_eq!(
expand_glob_braces(r"src/\{literal\}/[{}].txt").unwrap(),
[r"src/\{literal\}/[{}].txt"]
);
}
#[cfg(windows)]
#[test]
fn glob_braces_treat_windows_backslashes_as_separators() {
assert_eq!(
expand_glob_braces(r"C:\build\{debug,release}\*.exe").unwrap(),
[r"C:\build\debug\*.exe", r"C:\build\release\*.exe"]
);
}
#[test]
fn source_and_output_matchers_support_ordered_brace_globs() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let source_matcher = build_source_matcher(
root,
root,
&[
"{Cargo.toml,README.md}".to_string(),
"!README.md".to_string(),
"README.md".to_string(),
],
);
let output_matcher = build_output_matcher(
root,
&[
"{Cargo.toml,README.md}".to_string(),
"!README.md".to_string(),
],
)
.unwrap();
assert!(is_source(&source_matcher, &root.join("Cargo.toml")));
assert!(is_source(&source_matcher, &root.join("README.md")));
assert!(is_output(&output_matcher, &root.join("Cargo.toml"), false));
assert!(!is_output(&output_matcher, &root.join("README.md"), false));
}
#[test]
fn output_hash_supports_brace_globs() {
let root = tempfile::tempdir().unwrap();
fs::write(root.path().join("a.out"), "a").unwrap();
fs::write(root.path().join("b.out"), "b").unwrap();
let task = Task {
outputs: crate::task::task_sources::TaskOutputs::Files(vec!["{a,b}.out".to_string()]),
..Default::default()
};
assert!(compute_output_hash(&task, root.path()).unwrap().is_some());
}
#[test]
fn output_mtime_includes_selected_directories() {
let root = tempfile::tempdir().unwrap();
let dist = root.path().join("dist");
let output = dist.join("result.txt");
fs::create_dir(&dist).unwrap();
fs::write(&output, "result").unwrap();
let file_mtime = filetime::FileTime::from_unix_time(100, 0);
let directory_mtime = filetime::FileTime::from_unix_time(200, 0);
filetime::set_file_mtime(&output, file_mtime).unwrap();
filetime::set_file_mtime(&dist, directory_mtime).unwrap();
let modified = get_last_modified(root.path(), &["dist".to_string()])
.unwrap()
.unwrap();
assert_eq!(modified, SystemTime::from(directory_mtime));
}
#[test]
fn output_hash_allows_missing_excluded_static_paths() {
let root = tempfile::tempdir().unwrap();
let task = Task {
outputs: crate::task::task_sources::TaskOutputs::Files(vec![
"missing.txt".to_string(),
"!missing.txt".to_string(),
]),
..Default::default()
};
assert!(compute_output_hash(&task, root.path()).unwrap().is_some());
}
#[test]
fn output_hash_allows_glob_matches_that_are_all_excluded() {
let root = tempfile::tempdir().unwrap();
fs::create_dir(root.path().join("dist")).unwrap();
fs::write(root.path().join("dist/vendor.js"), "vendor").unwrap();
let task = Task {
outputs: crate::task::task_sources::TaskOutputs::Files(vec![
"dist/*.js".to_string(),
"!dist/vendor.js".to_string(),
]),
..Default::default()
};
assert!(compute_output_hash(&task, root.path()).unwrap().is_some());
}
#[test]
fn task_state_key_includes_all_definition_sources() {
let root = Path::new("/project");
let mut task = Task {
name: "build".to_string(),
config_source: PathBuf::from(".mise/tasks/build"),
..Default::default()
};
let primary_key = task_state_key(&task, root);
task.additional_config_sources
.push(PathBuf::from("mise.toml"));
assert_ne!(primary_key, task_state_key(&task, root));
}
#[test]
fn task_state_key_changes_when_run_changes() {
use crate::task::RunEntry;
let root = Path::new("/project");
let mut task = Task {
name: "build".to_string(),
config_source: PathBuf::from("mise.toml"),
run: vec![RunEntry::Script("echo v1".to_string())],
..Default::default()
};
let key_v1 = task_state_key(&task, root);
task.run = vec![RunEntry::Script("echo v2".to_string())];
assert_ne!(key_v1, task_state_key(&task, root));
}
#[test]
fn task_state_key_changes_when_sources_change() {
let root = Path::new("/project");
let mut task = Task {
name: "build".to_string(),
config_source: PathBuf::from("mise.toml"),
sources: vec!["src.txt".to_string()],
..Default::default()
};
let key_v1 = task_state_key(&task, root);
task.sources = vec!["other.txt".to_string()];
assert_ne!(key_v1, task_state_key(&task, root));
}
#[test]
fn glob_patterns_drops_excludes_and_unescapes() {
let inputs = vec![
"src/**/*.ts".to_string(),
"!src/**/*.test.ts".to_string(),
"\\!literal.txt".to_string(),
"tsconfig.json".to_string(),
];
assert_eq!(
source_glob_patterns(&inputs),
vec!["src/**/*.ts", "!literal.txt", "tsconfig.json"],
);
}
#[test]
fn matcher_includes_plain_pattern() {
assert!(matches(&["src/**/*.ts"], "src/foo.ts"));
assert!(matches(&["src/**/*.ts"], "src/sub/foo.ts"));
assert!(!matches(&["src/**/*.ts"], "lib/foo.ts"));
}
#[test]
fn matcher_negation_excludes() {
let pats = &["src/**/*.ts", "!src/**/*.test.ts"];
assert!(matches(pats, "src/foo.ts"));
assert!(!matches(pats, "src/foo.test.ts"));
}
#[test]
fn matcher_reincludes_after_negation() {
let pats = &["src/**/*.ts", "!src/**/*.test.ts", "src/keep.test.ts"];
assert!(matches(pats, "src/foo.ts"));
assert!(!matches(pats, "src/foo.test.ts"));
assert!(matches(pats, "src/keep.test.ts"));
}
#[test]
fn matcher_escaped_literal_bang() {
let pats = &["\\!important.txt", "!ignored.txt"];
assert!(matches(pats, "!important.txt"));
assert!(!matches(pats, "ignored.txt"));
}
#[test]
#[cfg(unix)]
fn matcher_absolute_literal_bang_under_root() {
let root = Path::new("/project");
let sources = vec!["/project/!important.txt".to_string()];
let matcher = build_source_matcher(root, root, &sources);
assert!(is_source(&matcher, Path::new("/project/!important.txt")));
assert!(!is_source(&matcher, Path::new("/project/other.txt")));
}
#[test]
#[cfg(unix)]
fn matcher_absolute_pattern_under_root() {
let root = Path::new("/proj");
let sources = vec!["/proj/input".to_string()];
let matcher = build_source_matcher(root, root, &sources);
assert!(is_source(&matcher, Path::new("/proj/input")));
assert!(!is_source(&matcher, Path::new("/proj/other")));
}
#[test]
#[cfg(unix)]
fn matcher_absolute_negation_under_root() {
let root = Path::new("/proj");
let sources = vec![
"/proj/src/**/*.ts".to_string(),
"!/proj/src/**/*.test.ts".to_string(),
];
let matcher = build_source_matcher(root, root, &sources);
assert!(is_source(&matcher, Path::new("/proj/src/foo.ts")));
assert!(!is_source(&matcher, Path::new("/proj/src/foo.test.ts")));
}
#[test]
#[cfg(unix)]
fn matcher_absolute_path_outside_root_passes_through() {
let root = Path::new("/proj");
let sources = vec!["/elsewhere/Cargo.toml".to_string()];
let matcher = build_source_matcher(root, root, &sources);
assert!(is_source(&matcher, Path::new("/elsewhere/Cargo.toml")));
}
#[test]
#[cfg(unix)]
fn matcher_subproject_absolute_workspace_pattern() {
let match_root = Path::new("/workspace");
let task_cwd = Path::new("/workspace/lib/worker");
let sources = vec!["/workspace/lib/**/*".to_string()];
let matcher = build_source_matcher(match_root, task_cwd, &sources);
assert!(is_source(
&matcher,
Path::new("/workspace/lib/worker/worker.go")
));
assert!(is_source(&matcher, Path::new("/workspace/lib/shared.go")));
assert!(!is_source(&matcher, Path::new("/workspace/other/file.go")));
}
#[test]
fn absolute_source_patterns_are_enumerated_from_a_subproject() -> Result<()> {
let temp = tempfile::tempdir()?;
let workspace = temp.path();
let task_cwd = workspace.join("packages/app");
let source = task_cwd.join("src/input.txt");
let global = workspace.join("workspace.txt");
fs::create_dir_all(source.parent().unwrap())?;
fs::write(&source, "source")?;
fs::write(&global, "global")?;
let sources = vec![
format!("{}/packages/app/src/**/*", workspace.display()),
global.to_string_lossy().to_string(),
];
let matcher = build_source_matcher(workspace, &task_cwd, &sources);
let metadatas = get_file_metadatas(&task_cwd, &source_glob_patterns(&sources), &matcher)?;
let paths = metadatas
.into_iter()
.map(|(path, _)| path)
.collect::<Vec<_>>();
assert!(paths.contains(&source), "{paths:?}");
assert!(paths.contains(&global), "{paths:?}");
Ok(())
}
#[test]
#[cfg(unix)]
fn matcher_subproject_relative_pattern() {
let match_root = Path::new("/workspace");
let task_cwd = Path::new("/workspace/lib/worker");
let sources = vec!["src/**/*.go".to_string()];
let matcher = build_source_matcher(match_root, task_cwd, &sources);
assert!(is_source(
&matcher,
Path::new("/workspace/lib/worker/src/main.go")
));
assert!(!is_source(&matcher, Path::new("/workspace/src/other.go")));
}
#[test]
fn content_hash_cache_reuses_unchanged_files() {
let tmp = tempfile::tempdir().unwrap();
let a = tmp.path().join("a.txt");
let b = tmp.path().join("b.txt");
std::fs::write(&a, "hello").unwrap();
std::fs::write(&b, "world").unwrap();
let metadatas = vec![
(a.clone(), a.metadata().unwrap()),
(b.clone(), b.metadata().unwrap()),
];
let mut cache = ContentHashCache::new();
let first = file_contents_to_hash(&metadatas, &mut cache).unwrap();
assert_eq!(cache.len(), 2);
let a_hash_v1 = cache.get(&a).unwrap().hash.clone();
let second = file_contents_to_hash(&metadatas, &mut cache).unwrap();
assert_eq!(first, second);
assert_eq!(cache.get(&a).unwrap().hash, a_hash_v1);
std::fs::write(&a, "hello world").unwrap();
let metadatas = vec![
(a.clone(), a.metadata().unwrap()),
(b.clone(), b.metadata().unwrap()),
];
let third = file_contents_to_hash(&metadatas, &mut cache).unwrap();
assert_ne!(second, third);
assert_ne!(cache.get(&a).unwrap().hash, a_hash_v1);
}
#[test]
fn content_hash_cache_prunes_dropped_files() {
let tmp = tempfile::tempdir().unwrap();
let a = tmp.path().join("a.txt");
let b = tmp.path().join("b.txt");
std::fs::write(&a, "hello").unwrap();
std::fs::write(&b, "world").unwrap();
let mut cache = ContentHashCache::new();
let metadatas = vec![
(a.clone(), a.metadata().unwrap()),
(b.clone(), b.metadata().unwrap()),
];
file_contents_to_hash(&metadatas, &mut cache).unwrap();
assert_eq!(cache.len(), 2);
let metadatas = vec![(a.clone(), a.metadata().unwrap())];
file_contents_to_hash(&metadatas, &mut cache).unwrap();
assert_eq!(cache.len(), 1);
assert!(cache.contains_key(&a));
assert!(!cache.contains_key(&b));
}
#[test]
fn content_hash_cache_round_trips_through_disk() {
let tmp = tempfile::tempdir().unwrap();
let a = tmp.path().join("a.txt");
std::fs::write(&a, "hello").unwrap();
let mut cache = ContentHashCache::new();
let metadatas = vec![(a.clone(), a.metadata().unwrap())];
file_contents_to_hash(&metadatas, &mut cache).unwrap();
let cache_path = tmp.path().join("cache.bin");
save_content_hash_cache(&cache_path, &cache).unwrap();
let loaded = load_content_hash_cache(&cache_path);
assert_eq!(loaded.len(), 1);
assert_eq!(loaded.get(&a).unwrap().hash, cache.get(&a).unwrap().hash,);
std::fs::write(&cache_path, b"not a valid msgpack stream").unwrap();
let loaded = load_content_hash_cache(&cache_path);
assert!(loaded.is_empty());
}
}