1use std::collections::{BTreeMap, BTreeSet};
2use std::ffi::OsString;
3use std::fs::File;
4use std::io::{Read, Write};
5use std::path::{Path, PathBuf};
6use std::sync::atomic::AtomicBool;
7use std::sync::{Arc, Mutex};
8use std::time::{Duration, Instant, SystemTime};
9
10use anyhow::{Context, Result, bail};
11use foldhash::{HashMap, HashMapExt, HashSet};
12use qk_config::Workspace;
13use qk_executor::{Capture, Display, Outcome, PreparedTask, execute_captured, read_capture};
14use qk_graph::ProjectGraph;
15use qk_lockfile::Lockfile;
16use qk_taskgraph::{Task, TaskGraph};
17use serde_json::{Value, json};
18
19use crate::ignored::Group;
20use crate::inputs::{
21 Declared, Inferred, JsonFields, Resolved, TsConfig, manifest_for_dependents,
22 workspace_without_resolution,
23};
24use crate::paths::{self, Outputs, Pattern};
25
26pub fn digest_file(path: &Path) -> Result<String> {
27 let mut hasher = blake3::Hasher::new();
28 let mut reader = File::open(path)?;
29 let mut buffer = [0; 64 * 1024];
30 loop {
31 let count = reader.read(&mut buffer)?;
32 if count == 0 {
33 break;
34 }
35 hasher.update(&buffer[..count]);
36 }
37 Ok(hasher.finalize().to_hex().to_string())
38}
39
40#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
43struct Stamp(Vec<i64>);
44
45impl Stamp {
46 fn new(metadata: &std::fs::Metadata) -> Self {
47 let modified = metadata
48 .modified()
49 .ok()
50 .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok())
51 .map_or(-1, |duration| duration.as_nanos() as i64);
52 #[allow(unused_mut)]
53 let mut values = vec![metadata.len() as i64, modified];
54 #[cfg(unix)]
55 {
56 use std::os::unix::fs::MetadataExt;
57 values.extend([
58 metadata.ctime(),
59 metadata.ctime_nsec(),
60 metadata.ino() as i64,
61 i64::from(metadata.mode()),
62 ]);
63 }
64 Self(values)
65 }
66
67 fn membership_settled(&self) -> bool {
70 #[cfg(unix)]
71 let width = 6;
72 #[cfg(not(unix))]
73 let width = 2;
74 self.0.chunks_exact(width).all(|values| {
75 let modified = SystemTime::UNIX_EPOCH + Duration::from_nanos(values[1].max(0) as u64);
76 let settled = |time| {
77 SystemTime::now()
78 .duration_since(time)
79 .is_ok_and(|age| age > Duration::from_secs(2))
80 };
81 if values[1] < 0 || !settled(modified) {
82 return false;
83 }
84 #[cfg(unix)]
85 {
86 let changed = SystemTime::UNIX_EPOCH
87 + Duration::new(values[2].max(0) as u64, values[3].max(0) as u32);
88 values[2] >= 0 && settled(changed)
89 }
90 #[cfg(not(unix))]
91 true
92 })
93 }
94
95 fn settled(&self) -> bool {
98 let modified = SystemTime::UNIX_EPOCH + Duration::from_nanos(self.0[1].max(0) as u64);
99 self.0[1] >= 0
100 && SystemTime::now()
101 .duration_since(modified)
102 .is_ok_and(|age| age > Duration::from_secs(2))
103 }
104}
105
106fn digests_path(root: &Path) -> PathBuf {
110 paths::worktree_state(root).join("digests.json")
111}
112
113#[derive(PartialEq, Eq, Hash)]
114struct RuntimeKey {
115 command: String,
116 environment: BTreeMap<OsString, OsString>,
117}
118
119pub struct Snapshot {
123 pub(crate) files: BTreeSet<String>,
124 extra_candidates: BTreeSet<String>,
125 membership: Mutex<Option<Arc<Membership>>>,
126 membership_prefixes: Mutex<HashMap<String, Arc<Vec<PathBuf>>>>,
127 source_exclusions: Vec<Outputs>,
128 scratch: Vec<Outputs>,
131 cache_relative: Option<PathBuf>,
132 projects: ProjectGraph,
133 roots: BTreeMap<String, String>,
135 generated_tasks: Vec<(Task, Outputs)>,
136 task_dependencies: BTreeMap<String, BTreeSet<String>>,
137 servers: BTreeMap<String, Vec<(String, Vec<Value>)>>,
139 source_ignore: Arc<SourceIgnore>,
140 canonical_root: PathBuf,
141 workspace_prefix: Option<PathBuf>,
142 patterns: Mutex<HashMap<(String, bool, bool), Arc<Pattern>>>,
143 digests: Mutex<HashMap<String, (Stamp, String)>>,
144 ignored_files: Mutex<BTreeSet<String>>,
147 digests_changed: std::sync::atomic::AtomicBool,
149 directories: Mutex<HashSet<PathBuf>>,
151 runtime: Mutex<HashMap<RuntimeKey, Arc<Mutex<Option<Value>>>>>,
153 lockfile: Mutex<Option<Arc<Installed>>>,
155 dependency_outputs: Mutex<HashMap<String, Arc<DependencyOutputs>>>,
158 outputs_written: std::sync::atomic::AtomicU64,
160}
161
162#[derive(Clone)]
165struct Membership {
166 files: Arc<BTreeSet<String>>,
167 source_ignore: Arc<SourceIgnore>,
168 stamps: BTreeMap<PathBuf, Option<Stamp>>,
169 directories: BTreeSet<PathBuf>,
170 directory_entries: BTreeMap<PathBuf, BTreeSet<(OsString, u8)>>,
171 root: PathBuf,
172 outputs: Vec<Outputs>,
173 output_anchors: BTreeMap<PathBuf, Vec<usize>>,
174 excluded_roots: BTreeSet<PathBuf>,
175 cache: Option<PathBuf>,
176 policies: BTreeSet<PathBuf>,
177 git_policies: BTreeSet<PathBuf>,
178 ignore_contents: BTreeMap<PathBuf, String>,
179 settled: bool,
180 listed: Option<Instant>,
182}
183
184#[derive(Clone)]
187struct Candidates {
188 files: Arc<BTreeSet<String>>,
189 source_ignore: Arc<SourceIgnore>,
190}
191
192impl Membership {
193 fn new(
195 root: &Path,
196 files: &BTreeSet<String>,
197 source_ignore: &Arc<SourceIgnore>,
198 outputs: &[Outputs],
199 cache: Option<&Path>,
200 git_policies: Option<&BTreeSet<PathBuf>>,
201 ) -> Result<Self> {
202 let excluded: BTreeSet<_> = outputs
203 .iter()
204 .flat_map(Outputs::complete_roots)
205 .map(PathBuf::from)
206 .collect();
207 let excluded_roots = excluded.clone();
208 let walk_root = root.to_owned();
209 let walk_cache = cache.map(PathBuf::from);
210 let mut directories = BTreeSet::new();
211 let mut stamps = BTreeMap::new();
212 for entry in ignore::WalkBuilder::new(root)
213 .hidden(false)
214 .parents(false)
215 .git_global(false)
216 .require_git(false)
217 .ignore(false)
218 .git_ignore(false)
219 .git_exclude(false)
220 .follow_links(false)
221 .filter_entry(move |entry| {
222 let relative = entry
223 .path()
224 .strip_prefix(&walk_root)
225 .unwrap_or(entry.path());
226 !matches!(
227 entry.file_name().to_str(),
228 Some(".git" | ".qk" | "node_modules")
229 ) && walk_cache
230 .as_ref()
231 .is_none_or(|cache| !relative.starts_with(cache))
232 && !relative.ancestors().any(|path| excluded.contains(path))
233 })
234 .build()
235 {
236 let entry = entry?;
237 if entry.file_type().is_some_and(|kind| kind.is_dir()) {
238 let directory = entry.path().to_owned();
239 stamps.insert(directory.clone(), membership_stamp(&directory)?);
240 directories.insert(directory);
241 }
242 }
243 for path in files {
246 for parent in Path::new(path).ancestors().skip(1) {
247 let directory = root.join(parent);
248 if directories.insert(directory.clone()) {
249 stamps.insert(directory.clone(), membership_stamp(&directory)?);
250 }
251 }
252 }
253 let mut policies = BTreeSet::new();
254 for directory in &directories {
255 for name in [".gitignore", ".nxignore"] {
256 let path = directory.join(name);
257 if membership_stamp(&path)?.is_some() {
260 policies.insert(path);
261 }
262 }
263 }
264 for directory in root.ancestors() {
265 policies.insert(directory.join(".gitignore"));
266 policies.insert(directory.join(".git"));
268 }
269 let git_policies = git_policies
270 .cloned()
271 .unwrap_or_else(|| git_membership_policies(root));
272 policies.extend(git_policies.iter().cloned());
273 for policy in &policies {
274 stamps.insert(policy.clone(), membership_stamp(policy)?);
275 }
276 let mut ignore_contents = BTreeMap::new();
279 for policy in &policies {
280 if matches!(
281 policy.file_name().and_then(|name| name.to_str()),
282 Some(".gitignore" | ".nxignore")
283 ) && let Some(content) = ignore_content(policy, &stamps[policy])?
284 {
285 ignore_contents.insert(policy.clone(), content);
286 }
287 }
288 let mut output_anchors: BTreeMap<PathBuf, Vec<usize>> = BTreeMap::new();
289 for (index, outputs) in outputs.iter().enumerate() {
290 for anchor in outputs.anchors() {
291 output_anchors
292 .entry(PathBuf::from(anchor))
293 .or_default()
294 .push(index);
295 }
296 }
297 let mut membership = Self {
298 files: Arc::new(BTreeSet::new()),
299 source_ignore: source_ignore.clone(),
300 stamps,
301 directories,
302 directory_entries: BTreeMap::new(),
303 root: root.to_owned(),
304 outputs: outputs.to_vec(),
305 output_anchors,
306 excluded_roots,
307 cache: cache.map(PathBuf::from),
308 policies,
309 git_policies,
310 ignore_contents,
311 settled: false,
312 listed: None,
313 };
314 for directory in &membership.directories {
315 if let Some(entries) = membership.entries(directory)? {
316 membership
317 .directory_entries
318 .insert(directory.clone(), entries);
319 }
320 }
321 Ok(membership)
322 }
323
324 fn entries(&self, directory: &Path) -> Result<Option<BTreeSet<(OsString, u8)>>> {
328 let entries = match std::fs::read_dir(directory) {
329 Ok(entries) => entries,
330 Err(error)
331 if matches!(
332 error.kind(),
333 std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
334 ) =>
335 {
336 return Ok(None);
337 }
338 Err(error) => return Err(error.into()),
339 };
340 let relative_directory = directory.strip_prefix(&self.root).unwrap_or(directory);
341 let mut outputs = BTreeSet::new();
342 for ancestor in relative_directory.ancestors() {
343 if let Some(indices) = self.output_anchors.get(ancestor) {
344 outputs.extend(indices.iter().copied());
345 }
346 }
347 for (_, indices) in self
348 .output_anchors
349 .range(relative_directory.to_owned()..)
350 .take_while(|(anchor, _)| anchor.starts_with(relative_directory))
351 {
352 outputs.extend(indices.iter().copied());
353 }
354 let mut names = BTreeSet::new();
355 for entry in entries {
356 let entry = entry?;
357 let name = entry.file_name();
358 let path = entry.path();
359 let relative = path.strip_prefix(&self.root).unwrap_or(&path);
360 if matches!(name.to_str(), Some(".git" | ".qk" | "node_modules"))
361 || self
362 .cache
363 .as_ref()
364 .is_some_and(|cache| relative.starts_with(cache))
365 || relative
366 .ancestors()
367 .any(|root| self.excluded_roots.contains(root))
368 {
369 continue;
370 }
371 if matches!(name.to_str(), Some(".gitignore" | ".nxignore"))
374 && !self.policies.contains(&path)
375 {
376 return Ok(None);
377 }
378 let kind = entry.file_type()?;
379 if kind.is_dir() && !self.directories.contains(&path) {
383 return Ok(None);
384 }
385 if !kind.is_dir()
386 && !matches!(name.to_str(), Some(".gitignore" | ".nxignore"))
387 && relative.to_str().is_some_and(|path| {
388 outputs
389 .iter()
390 .any(|index| self.outputs[*index].matches(path))
391 })
392 {
393 continue;
394 }
395 names.insert((
396 name,
397 if kind.is_dir() {
398 1
399 } else if kind.is_symlink() {
400 2
401 } else {
402 0
403 },
404 ));
405 }
406 Ok(Some(names))
407 }
408
409 fn state(&self, prefixes: Option<&[PathBuf]>) -> Result<(bool, bool)> {
413 let mut settled = true;
414 let selected = prefixes.map(|prefixes| {
415 let mut selected = BTreeSet::new();
416 for prefix in prefixes {
417 for ancestor in prefix.ancestors() {
418 if let Some(directory) = self.directories.get(ancestor) {
419 selected.insert(directory);
420 }
421 }
422 selected.extend(
423 self.directories
424 .range(prefix.clone()..)
425 .take_while(|directory| directory.starts_with(prefix)),
426 );
427 }
428 selected
429 });
430 let paths: Box<dyn Iterator<Item = (&PathBuf, bool)>> = match selected {
431 Some(selected) => Box::new(
432 selected
433 .into_iter()
434 .map(|path| (path, true))
435 .chain(self.policies.iter().map(|path| (path, false))),
436 ),
437 None => Box::new(
438 self.stamps
439 .keys()
440 .map(|path| (path, self.directories.contains(path))),
441 ),
442 };
443 for (path, directory) in paths {
444 let before = &self.stamps[path];
445 let after = membership_stamp(path)?;
446 if directory {
447 if after.is_some() && !self.directory_entries.contains_key(path) {
448 return Ok((false, false));
449 }
450 if *before != after
455 || after
456 .as_ref()
457 .is_some_and(|stamp| !stamp.membership_settled())
458 {
459 let entries = self.entries(path)?;
460 if entries.as_ref() != self.directory_entries.get(path)
461 || membership_stamp(path)? != after
462 {
463 return Ok((false, false));
464 }
465 }
466 } else {
467 let recent = after
468 .as_ref()
469 .is_some_and(|stamp| !stamp.membership_settled());
470 if (*before != after || recent)
471 && let Some(content) = self.ignore_contents.get(path)
472 {
473 if ignore_content(path, &after)?.as_ref() != Some(content) {
474 return Ok((false, false));
475 }
476 continue;
477 }
478 if *before != after {
479 return Ok((false, false));
480 }
481 settled &= !recent;
482 }
483 }
484 Ok((true, settled))
485 }
486
487 fn candidates(&self) -> Candidates {
489 Candidates {
490 files: self.files.clone(),
491 source_ignore: self.source_ignore.clone(),
492 }
493 }
494}
495
496fn git_membership_policies(root: &Path) -> BTreeSet<PathBuf> {
499 let mut policies = BTreeSet::new();
500 if let Ok(output) = paths::git(
501 root,
502 &[
503 "rev-parse",
504 "--path-format=absolute",
505 "--git-path",
506 "index",
507 "--git-path",
508 "info/exclude",
509 "--git-path",
510 "config",
511 "--git-path",
512 "config.worktree",
513 ],
514 ) && output.status.success()
515 && let Ok(paths) = String::from_utf8(output.stdout)
516 {
517 policies.extend(paths.lines().map(PathBuf::from));
518 }
519 if let Ok(output) = paths::git(root, &["config", "--null", "--show-origin", "--list"])
522 && output.status.success()
523 {
524 let output = String::from_utf8_lossy(&output.stdout);
528 let mut fields = output.split_terminator('\0');
529 while let (Some(origin), Some(entry)) = (fields.next(), fields.next()) {
530 if let Some(path) = origin.strip_prefix("file:") {
531 let origin = root.join(path);
532 if let Some((name, value)) = entry.split_once('\n')
533 && name.starts_with("include")
534 && name.ends_with(".path")
535 {
536 let included = if let Some(relative) = value.strip_prefix("~/") {
537 std::env::var_os("HOME")
538 .map(PathBuf::from)
539 .unwrap_or_default()
540 .join(relative)
541 } else {
542 origin.parent().unwrap_or(root).join(value)
543 };
544 policies.insert(included);
547 }
548 policies.insert(origin);
549 }
550 }
551 }
552 if let Ok(output) = paths::git(
553 root,
554 &["config", "--null", "--path", "--get", "core.excludesfile"],
555 ) && output.status.success()
556 && let Ok(path) = String::from_utf8(output.stdout)
557 {
558 policies.insert(root.join(path.strip_suffix('\0').unwrap_or(&path)));
559 }
560 for variable in ["GIT_CONFIG_SYSTEM", "GIT_CONFIG_GLOBAL"] {
561 if let Some(path) = std::env::var_os(variable) {
562 policies.insert(PathBuf::from(path));
563 }
564 }
565 policies.insert(PathBuf::from("/etc/gitconfig"));
566 if let Some(home) = std::env::var_os("HOME") {
567 let home = PathBuf::from(home);
568 policies.insert(home.join(".gitconfig"));
569 let config = std::env::var_os("XDG_CONFIG_HOME")
570 .map(PathBuf::from)
571 .unwrap_or_else(|| home.join(".config"));
572 policies.insert(config.join("git/config"));
573 policies.insert(config.join("git/ignore"));
574 }
575 policies
576}
577
578fn ignore_content(path: &Path, expected: &Option<Stamp>) -> Result<Option<String>> {
581 if expected.is_none() || membership_stamp(path)? != *expected {
582 return Ok(None);
583 }
584 match std::fs::symlink_metadata(path) {
585 Ok(metadata) if metadata.is_file() => {}
586 Ok(_) => return Ok(None),
587 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
588 Err(error) => return Err(error.into()),
589 }
590 let content = match std::fs::read(path) {
591 Ok(content) => content,
592 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
593 Err(error) => return Err(error.into()),
594 };
595 Ok((membership_stamp(path)? == *expected).then(|| blake3::hash(&content).to_hex().to_string()))
596}
597
598fn membership_stamp(path: &Path) -> Result<Option<Stamp>> {
600 match std::fs::symlink_metadata(path) {
601 Ok(metadata) => {
602 let mut stamp = Stamp::new(&metadata);
603 if metadata.is_symlink() {
604 match std::fs::metadata(path) {
607 Ok(target) => stamp.0.extend(Stamp::new(&target).0),
608 Err(error) if error.kind() == std::io::ErrorKind::NotFound => stamp.0[1] = -1,
609 Err(error) => return Err(error.into()),
610 }
611 }
612 Ok(Some(stamp))
613 }
614 Err(error)
615 if matches!(
616 error.kind(),
617 std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
618 ) =>
619 {
620 Ok(None)
621 }
622 Err(error) => Err(error).with_context(|| format!("cannot inspect {}", path.display())),
623 }
624}
625
626struct DependencyOutputs {
629 anchors: Vec<String>,
630 files: Vec<String>,
631 values: Mutex<HashMap<String, Value>>,
632}
633
634struct Installed {
637 content: String,
638 lockfile: Option<Lockfile>,
640 digests: Mutex<HashMap<String, String>>,
641}
642
643impl Installed {
644 fn digest(&self, key: &str, fingerprints: impl FnOnce() -> BTreeSet<String>) -> String {
645 if let Some(digest) = self.digests.lock().unwrap().get(key) {
646 return digest.clone();
647 }
648 let mut hasher = blake3::Hasher::new();
649 for fingerprint in fingerprints() {
650 hasher.update(fingerprint.as_bytes());
651 hasher.update(b"\n");
652 }
653 let digest = hasher.finalize().to_hex().to_string();
654 self.digests
655 .lock()
656 .unwrap()
657 .insert(key.to_owned(), digest.clone());
658 digest
659 }
660}
661
662impl Snapshot {
663 pub fn new(workspace: &Workspace, graph: &TaskGraph, cache_path: &Path) -> Result<Self> {
664 let canonical_root = workspace.root.canonicalize()?;
667 let (files, lockfile, (digests, workspace_prefix)) = std::thread::scope(|scope| {
668 let lockfile = scope.spawn(|| parse_lockfile(&workspace.root.join("pnpm-lock.yaml")));
669 let digests = scope.spawn(|| {
670 let digests = load_digests(&workspace.root);
671 let workspace_prefix = paths::git_path(&workspace.root, "--show-toplevel")
672 .and_then(|root| root.canonicalize().ok())
673 .and_then(|root| canonical_root.strip_prefix(root).ok().map(PathBuf::from));
674 (digests, workspace_prefix)
675 });
676 let files = source_files(&workspace.root);
677 (
678 files,
679 lockfile.join().expect("lockfile thread panicked"),
680 digests.join().expect("digest thread panicked"),
681 )
682 });
683 let mut files = files?;
684 let cache_relative = cache_path
685 .strip_prefix(&workspace.root)
686 .ok()
687 .map(PathBuf::from);
688 let mut source_exclusions: Vec<_> = graph
691 .tasks
692 .values()
693 .filter_map(|task| Outputs::new(workspace, task).ok())
694 .filter(Outputs::is_explicit)
695 .collect();
696 for task in graph.tasks.values() {
697 let Ok(entries) = crate::warm::config(workspace, task) else {
698 continue;
699 };
700 if let Ok(paths) = Outputs::from_paths(&crate::warm::kept_paths(&entries)) {
701 source_exclusions.push(paths);
702 }
703 }
704 let mut scratch_paths = Vec::new();
707 for task in graph.tasks.values() {
708 let scratch: Vec<String> = crate::scratch::directories(workspace, task)
709 .unwrap_or_default()
710 .iter()
711 .filter_map(|directory| {
712 let relative = directory.strip_prefix(&workspace.root).ok()?.to_str()?;
713 let named = crate::glob::escape(&relative.replace('\\', "/"));
714 Some([named.clone(), format!("{named}/**")])
715 })
716 .flatten()
717 .collect();
718 if !scratch.is_empty()
719 && let Ok(paths) = Outputs::from_paths(&scratch)
720 {
721 source_exclusions.push(paths.clone());
722 scratch_paths.push(paths);
723 }
724 }
725 exclude_artifacts(&mut files, &source_exclusions, cache_relative.as_deref());
726 Ok(Self {
727 files,
728 extra_candidates: BTreeSet::new(),
729 membership: Mutex::default(),
730 membership_prefixes: Mutex::default(),
731 source_exclusions,
732 scratch: scratch_paths,
733 cache_relative,
734 projects: ProjectGraph::build(workspace)?,
735 roots: workspace
736 .projects
737 .values()
738 .map(|project| (project.root.clone(), project.name.clone()))
739 .collect(),
740 source_ignore: Arc::new(SourceIgnore::new(&workspace.root)?),
741 generated_tasks: graph
742 .tasks
743 .values()
744 .filter_map(|task| {
745 let outputs = Outputs::new(workspace, task).ok()?;
746 outputs.is_explicit().then(|| (task.clone(), outputs))
747 })
748 .collect(),
749 task_dependencies: graph
750 .tasks
751 .iter()
752 .map(|(id, task)| (id.clone(), task.dependencies.clone()))
753 .collect(),
754 servers: graph
755 .tasks
756 .keys()
757 .map(|id| (id.clone(), servers(graph, id)))
758 .collect(),
759 canonical_root,
760 workspace_prefix,
761 patterns: Mutex::default(),
762 digests: Mutex::new(digests),
763 digests_changed: Default::default(),
764 ignored_files: Mutex::default(),
765 directories: Mutex::default(),
766 runtime: Mutex::default(),
767 lockfile: Mutex::new(lockfile),
768 dependency_outputs: Mutex::default(),
769 outputs_written: Default::default(),
770 })
771 }
772
773 fn membership_prefixes(&self, workspace: &Workspace, task: &Task) -> Result<Arc<Vec<PathBuf>>> {
775 let mut prefixes = self.membership_prefixes.lock().unwrap();
776 if let Some(prefixes) = prefixes.get(&task.id) {
777 return Ok(prefixes.clone());
778 }
779 let empty = BTreeSet::new();
780 let cancelled = AtomicBool::new(false);
781 let mut resolver = Resolver::new(self, workspace, None, &cancelled, &empty);
782 resolver.task_inputs(task)?;
783 let paths = Arc::new(
784 resolver
785 .scopes
786 .values()
787 .flat_map(|scope| &scope.patterns)
788 .filter(|pattern| !pattern.excluded)
789 .map(|pattern| &pattern.prefix)
790 .chain(resolver.json_globs.iter().map(|glob| &glob.prefix))
791 .map(|prefix| workspace.root.join(prefix))
792 .collect(),
793 );
794 prefixes.insert(task.id.clone(), Arc::clone(&paths));
795 Ok(paths)
796 }
797
798 #[cfg(test)]
801 fn current_files(&self, root: &Path) -> Result<Arc<BTreeSet<String>>> {
802 Ok(self.current_files_for(root, None, "test")?.files)
803 }
804
805 fn current_files_for(
807 &self,
808 root: &Path,
809 prefixes: Option<&[PathBuf]>,
810 task: &str,
811 ) -> Result<Candidates> {
812 let requested = Instant::now();
813 loop {
814 let previous = self.membership.lock().unwrap().clone();
815 let (unchanged, settled) = match &previous {
818 Some(entry) => entry.state(prefixes)?,
819 None => (false, false),
820 };
821 if settled && previous.as_ref().unwrap().settled {
822 return Ok(previous.as_ref().unwrap().candidates());
823 }
824 let waiting = crate::profile::span(task, "inventory_wait");
827 let mut cached = self.membership.lock().unwrap();
828 drop(waiting);
829 if let Some(entry) = &*cached
834 && entry.listed.is_some_and(|listed| listed >= requested)
835 {
836 return Ok(entry.candidates());
837 }
838 let current = match (&previous, &*cached) {
839 (Some(previous), Some(current)) => Arc::ptr_eq(previous, current),
840 (None, None) => true,
841 _ => false,
842 };
843 if !current {
844 continue;
847 }
848 drop(previous);
849 let _listing = crate::profile::span(task, "inventory_list");
850 let listed = Instant::now();
851 let rebuilt = !unchanged || settled;
856 let mut membership = if !rebuilt {
857 Arc::unwrap_or_clone(cached.take().unwrap())
858 } else {
859 Membership::new(
862 root,
863 cached.as_ref().map_or(&self.files, |entry| &entry.files),
864 &self.source_ignore,
865 &self.source_exclusions,
866 self.cache_relative.as_deref(),
867 cached
868 .as_ref()
869 .filter(|entry| unchanged && entry.settled)
870 .map(|entry| &entry.git_policies),
871 )?
872 };
873 let mut files = source_files(root)?;
874 exclude_artifacts(
875 &mut files,
876 &self.source_exclusions,
877 self.cache_relative.as_deref(),
878 );
879 let ignore = SourceIgnore::new(root)?;
880 files.extend(
881 self.extra_candidates
882 .iter()
883 .filter(|path| !ignore.matches(path))
884 .cloned(),
885 );
886 membership.source_ignore = Arc::new(ignore);
887 membership.files = Arc::new(files);
888 let candidates = membership.candidates();
889 let covered = candidates.files.iter().all(|path| {
892 Path::new(path)
893 .ancestors()
894 .skip(1)
895 .all(|parent| membership.stamps.contains_key(&root.join(parent)))
896 });
897 membership.settled = rebuilt && covered && membership.state(None)?.1;
898 membership.listed = Some(listed);
899 *cached = Some(Arc::new(membership));
900 return Ok(candidates);
901 }
902 }
903
904 pub fn save_digests(&self, root: &Path) {
906 if !self
907 .digests_changed
908 .load(std::sync::atomic::Ordering::Relaxed)
909 {
910 return;
911 }
912 let digests = self.digests.lock().unwrap();
913 let ignored = self.ignored_files.lock().unwrap();
914 let entries: BTreeMap<&String, (&Stamp, &String)> = digests
915 .iter()
916 .filter(|(path, _)| self.files.contains(*path) || ignored.contains(*path))
917 .map(|(path, (stamp, digest))| (path, (stamp, digest)))
918 .collect();
919 let path = digests_path(root);
920 let saved = (|| -> Result<()> {
921 let directory = path.parent().context("digests have a directory")?;
922 std::fs::create_dir_all(directory)?;
923 let mut file = tempfile::NamedTempFile::new_in(directory)?;
924 {
925 let mut writer = std::io::BufWriter::new(file.as_file_mut());
926 serde_json::to_writer(&mut writer, &json!({"version": 1, "entries": entries}))?;
927 writer.flush()?;
928 }
929 file.persist(&path)?;
930 Ok(())
931 })();
932 if let Err(error) = saved {
933 qk_executor::status!("qk: could not save file digests: {error:#}");
934 }
935 }
936
937 fn safe_parents(&self, root: &Path, path: &str) -> Result<()> {
939 paths::validate_path(path)?;
940 let mut current = root.to_owned();
941 for part in Path::new(path)
942 .parent()
943 .into_iter()
944 .flat_map(Path::components)
945 {
946 current.push(part);
947 if self.directories.lock().unwrap().contains(¤t) {
948 continue;
949 }
950 match std::fs::symlink_metadata(¤t) {
951 Ok(metadata) if metadata.file_type().is_symlink() => {
952 bail!("input parent is a symlink: {}", current.display())
953 }
954 Ok(metadata) if !metadata.is_dir() => {
955 bail!("input parent is not a directory: {}", current.display())
956 }
957 Ok(_) => {
958 self.directories.lock().unwrap().insert(current.clone());
959 }
960 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
961 Err(error) => return Err(error.into()),
962 }
963 }
964 Ok(())
965 }
966
967 fn installed(&self, root: &Path, task: &str) -> Result<Option<Arc<Installed>>> {
968 let absolute = root.join("pnpm-lock.yaml");
969 let Ok(metadata) = std::fs::symlink_metadata(&absolute) else {
970 return Ok(None);
971 };
972 if !metadata.is_file() {
973 return Ok(None);
974 }
975 let content = self.digest("pnpm-lock.yaml", &absolute, &metadata)?;
976 let mut current = self.lockfile.lock().unwrap();
977 if let Some(installed) = &*current
978 && installed.content == content
979 {
980 return Ok(Some(installed.clone()));
981 }
982 let parsing = crate::profile::span(task, "lockfile_parse");
983 let parsed = parse_lockfile(&absolute);
984 drop(parsing);
985 let Some(installed) = parsed else {
986 return Ok(None);
987 };
988 *current = Some(installed.clone());
989 Ok(Some(installed))
990 }
991
992 pub fn with_candidates(mut self, paths: &[String]) -> Self {
995 self.files.extend(
996 paths
997 .iter()
998 .filter(|path| !self.source_ignore.matches(path))
999 .cloned(),
1000 );
1001 self.extra_candidates.extend(paths.iter().cloned());
1002 self
1003 }
1004
1005 fn pattern(&self, pattern: &str, negated: bool) -> Result<Arc<Pattern>> {
1006 self.compiled(pattern, negated, false)
1007 }
1008
1009 fn fileset_pattern(&self, pattern: &str, negated: bool) -> Result<Arc<Pattern>> {
1011 self.compiled(pattern, negated, true)
1012 }
1013
1014 fn compiled(&self, pattern: &str, negated: bool, fileset: bool) -> Result<Arc<Pattern>> {
1015 let key = (pattern.to_owned(), negated, fileset);
1016 if let Some(pattern) = self.patterns.lock().unwrap().get(&key) {
1017 return Ok(pattern.clone());
1018 }
1019 let compiled = Arc::new(if fileset {
1020 Pattern::fileset(pattern, negated)?
1021 } else {
1022 Pattern::new(pattern, negated)?
1023 });
1024 self.patterns.lock().unwrap().insert(key, compiled.clone());
1025 Ok(compiled)
1026 }
1027
1028 fn digest(&self, path: &str, absolute: &Path, metadata: &std::fs::Metadata) -> Result<String> {
1029 let stamp = Stamp::new(metadata);
1030 if let Some((known, digest)) = self.digests.lock().unwrap().get(path)
1031 && *known == stamp
1032 {
1033 return Ok(digest.clone());
1034 }
1035 let digest = digest_file(absolute)?;
1036 let after = Stamp::new(&std::fs::symlink_metadata(absolute)?);
1038 if after == stamp && stamp.settled() {
1039 self.digests
1040 .lock()
1041 .unwrap()
1042 .insert(path.to_owned(), (stamp, digest.clone()));
1043 self.digests_changed
1044 .store(true, std::sync::atomic::Ordering::Relaxed);
1045 }
1046 Ok(digest)
1047 }
1048
1049 fn dependency_outputs(
1052 &self,
1053 workspace: &Workspace,
1054 task: &Task,
1055 ) -> Result<Arc<DependencyOutputs>> {
1056 if let Some(outputs) = self.dependency_outputs.lock().unwrap().get(&task.id) {
1057 return Ok(outputs.clone());
1058 }
1059 let written = self
1060 .outputs_written
1061 .load(std::sync::atomic::Ordering::SeqCst);
1062 let outputs = Arc::new(self.fresh_dependency_outputs(workspace, task)?);
1063 let mut listed = self.dependency_outputs.lock().unwrap();
1064 if self
1065 .outputs_written
1066 .load(std::sync::atomic::Ordering::SeqCst)
1067 != written
1068 {
1069 return Ok(outputs);
1070 }
1071 Ok(listed.entry(task.id.clone()).or_insert(outputs).clone())
1072 }
1073
1074 fn fresh_dependency_outputs(
1076 &self,
1077 workspace: &Workspace,
1078 task: &Task,
1079 ) -> Result<DependencyOutputs> {
1080 let declared = Outputs::new(workspace, task)?;
1081 let mut files = Vec::new();
1082 for entry in declared.entries(&workspace.root)? {
1083 let (path, metadata) = entry?;
1084 if !metadata.is_dir() {
1085 files.push(path);
1086 }
1087 }
1088 Ok(DependencyOutputs {
1089 anchors: declared.anchors().map(str::to_owned).collect(),
1090 files,
1091 values: Mutex::default(),
1092 })
1093 }
1094
1095 pub(crate) fn outputs_written(&self, workspace: &Workspace, task: &Task) {
1098 let anchors: Option<Vec<String>> = Outputs::new(workspace, task)
1099 .ok()
1100 .filter(Outputs::is_explicit)
1101 .map(|outputs| outputs.anchors().map(str::to_owned).collect());
1102 let overlap = |a: &str, b: &str| {
1103 a == b
1104 || b.strip_prefix(a).is_some_and(|rest| rest.starts_with('/'))
1105 || a.strip_prefix(b).is_some_and(|rest| rest.starts_with('/'))
1106 };
1107 let mut listed = self.dependency_outputs.lock().unwrap();
1108 self.outputs_written
1109 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1110 listed.retain(|_, outputs| {
1111 anchors.as_ref().is_some_and(|written| {
1112 !written
1113 .iter()
1114 .any(|a| outputs.anchors.iter().any(|b| overlap(a, b)))
1115 })
1116 });
1117 }
1118
1119 fn ignored_value(&self, root: &Path, path: &str) -> Result<Option<Value>> {
1122 let absolute = root.join(path);
1123 let metadata = match std::fs::metadata(&absolute) {
1124 Ok(metadata) if metadata.is_file() => metadata,
1125 Ok(_) => return Ok(None),
1126 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1127 Err(error) => return Err(error.into()),
1128 };
1129 let mode = crate::store::mode(&metadata) & 0o111 != 0;
1130 self.ignored_files.lock().unwrap().insert(path.to_owned());
1131 Ok(Some(
1132 json!({"content":self.digest(path, &absolute, &metadata)?, "mode":mode}),
1133 ))
1134 }
1135
1136 pub(crate) fn recorded_value(&self, root: &Path, path: &str) -> Result<Value> {
1138 self.file_value(root, path)
1139 }
1140
1141 fn file_value(&self, root: &Path, path: &str) -> Result<Value> {
1142 self.file_value_with_links(root, path, &mut BTreeSet::new())
1143 }
1144
1145 fn file_value_with_links(
1146 &self,
1147 root: &Path,
1148 path: &str,
1149 visiting: &mut BTreeSet<PathBuf>,
1150 ) -> Result<Value> {
1151 self.safe_parents(root, path)?;
1152 let absolute = root.join(path);
1153 let metadata = std::fs::symlink_metadata(&absolute)?;
1154 if metadata.file_type().is_symlink() {
1155 let target = std::fs::read_link(&absolute)?;
1156 let Ok(resolved) = absolute.canonicalize() else {
1158 return Ok(json!({"link":target, "dangling":true}));
1159 };
1160 let Ok(relative) = resolved.strip_prefix(&self.canonical_root) else {
1161 bail!("input symlink resolves outside the workspace: {path}");
1162 };
1163 if resolved.is_file() {
1164 let metadata = std::fs::metadata(&resolved)?;
1165 let key = paths::relative(Path::new(""), relative)?;
1166 return Ok(
1167 json!({"link":target, "content":self.digest(&key, &resolved, &metadata)?,
1168 "mode":crate::store::mode(&metadata) & 0o111 != 0}),
1169 );
1170 }
1171 let relative = paths::relative(Path::new(""), relative)?;
1172 if relative.is_empty() {
1173 bail!("input symlink resolves to the workspace root: {path}");
1174 }
1175 if !visiting.insert(resolved.clone()) {
1176 bail!("directory input symlink cycle: {path}");
1177 }
1178 let mut contents = BTreeMap::new();
1179 for entry in walkdir::WalkDir::new(&resolved).follow_links(false) {
1182 let entry = entry?;
1183 if entry.file_type().is_dir() {
1184 continue;
1185 }
1186 let file = paths::relative(&self.canonical_root, entry.path())?;
1187 let value = self.file_value_with_links(&self.canonical_root, &file, visiting)?;
1188 contents.insert(
1189 file.strip_prefix(&relative).unwrap_or(&file).to_owned(),
1190 value,
1191 );
1192 }
1193 visiting.remove(&resolved);
1194 return Ok(json!({"link":target, "directory":contents}));
1195 }
1196 if !metadata.is_file() {
1197 bail!("input is not a regular file: {path}");
1198 }
1199 let mode = crate::store::mode(&metadata) & 0o111 != 0;
1202 Ok(json!({"content":self.digest(path, &absolute, &metadata)?, "mode":mode}))
1203 }
1204}
1205
1206fn parse_lockfile(path: &Path) -> Option<Arc<Installed>> {
1209 let text = std::fs::read_to_string(path).ok()?;
1210 let lockfile = match Lockfile::parse(&text) {
1211 Ok(lockfile) => Some(lockfile),
1212 Err(error) => {
1213 qk_executor::status!("qk: pnpm-lock.yaml: keying tasks by the whole file ({error:#})");
1214 None
1215 }
1216 };
1217 Some(Arc::new(Installed {
1218 content: blake3::hash(text.as_bytes()).to_hex().to_string(),
1219 lockfile,
1220 digests: Mutex::default(),
1221 }))
1222}
1223
1224fn load_digests(root: &Path) -> HashMap<String, (Stamp, String)> {
1225 #[derive(serde::Deserialize)]
1226 struct Saved {
1227 version: u32,
1228 entries: HashMap<String, (Stamp, String)>,
1229 }
1230 std::fs::read(digests_path(root))
1231 .ok()
1232 .and_then(|bytes| serde_json::from_slice::<Saved>(&bytes).ok())
1233 .filter(|saved| saved.version == 1)
1234 .map(|saved| saved.entries)
1235 .unwrap_or_default()
1236}
1237
1238fn exclude_artifacts(files: &mut BTreeSet<String>, outputs: &[Outputs], cache: Option<&Path>) {
1241 let mut generated = BTreeSet::new();
1242 for outputs in outputs {
1243 for anchor in outputs.anchors() {
1244 generated.extend(
1245 under(files, anchor)
1246 .filter(|path| outputs.matches(path))
1247 .cloned(),
1248 );
1249 }
1250 }
1251 files.retain(|path| {
1252 !generated.contains(path) && cache.is_none_or(|cache| !Path::new(path).starts_with(cache))
1253 });
1254}
1255
1256impl Snapshot {
1258 fn owner(&self, path: &str) -> Option<&str> {
1261 let mut path = path;
1262 loop {
1263 if let Some(project) = self.roots.get(path) {
1264 return Some(project);
1265 }
1266 match path.rfind('/') {
1267 Some(slash) => path = &path[..slash],
1268 None if path != "." => path = ".",
1269 None => return None,
1270 }
1271 }
1272 }
1273}
1274
1275fn under<'a>(files: &'a BTreeSet<String>, prefix: &'a str) -> impl Iterator<Item = &'a String> {
1276 files
1277 .range::<str, _>((
1278 std::ops::Bound::Included(prefix),
1279 std::ops::Bound::Unbounded,
1280 ))
1281 .take_while(move |path| path.starts_with(prefix))
1282 .filter(move |path| path.len() == prefix.len() || path.as_bytes()[prefix.len()] == b'/')
1283}
1284
1285pub struct SourceIgnore(ignore::gitignore::Gitignore);
1287
1288impl SourceIgnore {
1289 pub fn new(root: &Path) -> Result<Self> {
1291 let mut builder = ignore::gitignore::GitignoreBuilder::new(root);
1292 for name in [".gitignore", ".nxignore"] {
1293 let path = root.join(name);
1294 if path.is_file()
1295 && let Some(error) = builder.add(path)
1296 {
1297 return Err(error.into());
1298 }
1299 }
1300 Ok(Self(builder.build()?))
1301 }
1302
1303 fn matches_directory(&self, path: &str) -> bool {
1304 self.matches(path) || self.0.matched(path, true).is_ignore()
1305 }
1306
1307 pub fn matches(&self, path: &str) -> bool {
1309 Path::new(path)
1311 .ancestors()
1312 .skip(1)
1313 .take_while(|parent| !parent.as_os_str().is_empty())
1314 .any(|parent| self.0.matched(parent, true).is_ignore())
1315 || self.0.matched(path, false).is_ignore()
1316 }
1317}
1318
1319pub fn source_files(root: &Path) -> Result<BTreeSet<String>> {
1321 let lines =
1322 |output: std::io::Result<std::process::Output>| -> Option<Result<BTreeSet<String>>> {
1323 let output = output.ok().filter(|output| output.status.success())?;
1324 Some(
1325 output
1326 .stdout
1327 .split(|byte| *byte == 0)
1328 .filter(|path| !path.is_empty())
1329 .map(|path| {
1330 String::from_utf8(path.to_vec()).context("input paths must be UTF-8")
1331 })
1332 .collect(),
1333 )
1334 };
1335 let (listed, deleted) = if root.join(".nxignore").is_file() {
1340 (None, None)
1343 } else {
1344 std::thread::scope(|scope| {
1345 let deleted =
1346 scope.spawn(|| paths::git(root, &["ls-files", "--deleted", "-z", "--", "."]));
1347 let listed = paths::git(
1348 root,
1349 &[
1350 "ls-files",
1351 "-v",
1352 "--cached",
1353 "--others",
1354 "--exclude-standard",
1355 "-z",
1356 "--",
1357 ".",
1358 ],
1359 );
1360 (
1361 lines(listed),
1362 lines(deleted.join().expect("git listing thread panicked")),
1363 )
1364 })
1365 };
1366 let candidates = match (listed, deleted) {
1367 (Some(listed), Some(deleted)) => {
1368 let deleted = deleted?;
1369 let mut files = BTreeSet::new();
1370 for entry in listed? {
1371 let Some((tag, path)) = entry.split_once(' ') else {
1372 bail!("unexpected git ls-files entry {entry:?}");
1373 };
1374 let absent =
1375 matches!(tag, "S" | "s") && std::fs::symlink_metadata(root.join(path)).is_err();
1376 if !absent && !deleted.contains(path) {
1377 files.insert(path.to_owned());
1378 }
1379 }
1380 files
1381 }
1382 _ => {
1383 let mut files = BTreeSet::new();
1384 for entry in ignore::WalkBuilder::new(root)
1385 .hidden(false)
1386 .parents(false)
1387 .git_global(false)
1388 .require_git(false)
1389 .ignore(false)
1390 .add_custom_ignore_filename(".nxignore")
1391 .follow_links(false)
1392 .filter_entry(|entry| {
1393 !matches!(
1394 entry.file_name().to_str(),
1395 Some(".git" | ".qk" | "node_modules")
1396 )
1397 })
1398 .build()
1399 {
1400 let entry = entry?;
1401 if entry.file_type().is_some_and(|kind| !kind.is_dir()) {
1402 files.insert(paths::relative(root, entry.path())?);
1403 }
1404 }
1405 files
1406 }
1407 };
1408 let ignore = SourceIgnore::new(root)?;
1409 Ok(candidates
1410 .into_iter()
1411 .filter(|path| {
1412 !ignore.matches(path)
1413 && !path
1414 .split('/')
1415 .any(|part| matches!(part, ".git" | ".qk" | "node_modules"))
1416 })
1417 .collect())
1418}
1419
1420pub(crate) fn json_projection(path: &str, text: &str, fields: &JsonFields) -> Option<Value> {
1424 let mut value: Value = if path.ends_with(".yaml") || path.ends_with(".yml") {
1425 serde_yaml_ng::from_str(text).ok()?
1426 } else {
1427 jsonc_parser::parse_to_serde_value(text, &Default::default()).ok()?
1428 };
1429 if let Some(fields) = &fields.fields {
1430 let mut kept = json!({});
1431 for field in fields {
1432 if let Some(found) = json_path(&value, field) {
1433 set_json_path(&mut kept, field, found.clone());
1434 }
1435 }
1436 value = kept;
1437 }
1438 for field in &fields.exclude {
1439 remove_json_path(&mut value, field);
1440 }
1441 Some(value)
1442}
1443
1444fn json_path<'v>(value: &'v Value, path: &str) -> Option<&'v Value> {
1445 path.split('.')
1446 .try_fold(value, |value, part| value.get(part))
1447}
1448
1449fn set_json_path(target: &mut Value, path: &str, value: Value) {
1450 let mut parts = path.split('.').peekable();
1451 let mut current = target;
1452 while let Some(part) = parts.next() {
1453 let Value::Object(object) = current else {
1454 return;
1455 };
1456 if parts.peek().is_none() {
1457 object.insert(part.to_owned(), value);
1458 return;
1459 }
1460 current = object.entry(part).or_insert_with(|| json!({}));
1461 }
1462}
1463
1464fn remove_json_path(target: &mut Value, path: &str) {
1465 let (parent, last) = match path.rsplit_once('.') {
1466 Some((parent, last)) => (Some(parent), last),
1467 None => (None, path),
1468 };
1469 let parent = match parent {
1470 Some(parent) => parent
1471 .split('.')
1472 .try_fold(&mut *target, |value, part| value.get_mut(part)),
1473 None => Some(target),
1474 };
1475 if let Some(Value::Object(object)) = parent {
1476 object.remove(last);
1477 }
1478}
1479
1480struct FilePattern {
1481 matcher: Arc<Pattern>,
1482 prefix: String,
1483 rest: Option<String>,
1486 excluded: bool,
1487 owner: Option<String>,
1489}
1490
1491#[derive(Default)]
1495pub struct Selection {
1496 scopes: Vec<Vec<FilePattern>>,
1497}
1498
1499impl Selection {
1500 fn selects(&self, snapshot: &Snapshot, path: &str) -> bool {
1504 let applies = |pattern: &FilePattern| {
1505 pattern.matcher.is_match(path)
1506 && (pattern.owner.is_none() || snapshot.owner(path) == pattern.owner.as_deref())
1507 };
1508 self.scopes.iter().any(|patterns| {
1509 patterns
1510 .iter()
1511 .any(|pattern| !pattern.excluded && applies(pattern))
1512 && !patterns
1513 .iter()
1514 .any(|pattern| pattern.excluded && applies(pattern))
1515 })
1516 }
1517
1518 fn selects_every_entry(&self, snapshot: &Snapshot, directory: &str) -> bool {
1523 let within = |prefix: &str| {
1524 prefix.is_empty()
1525 || directory == prefix
1526 || directory
1527 .strip_prefix(prefix)
1528 .is_some_and(|rest| rest.starts_with('/'))
1529 };
1530 let reaches = |prefix: &str| {
1531 within(prefix)
1532 || directory.is_empty()
1533 || prefix
1534 .strip_prefix(directory)
1535 .is_some_and(|rest| rest.starts_with('/'))
1536 };
1537 let owner = snapshot.owner(if directory.is_empty() { "." } else { directory });
1538 self.scopes.iter().any(|patterns| {
1539 patterns.iter().any(|pattern| {
1540 !pattern.excluded
1541 && within(&pattern.prefix)
1542 && matches!(pattern.rest.as_deref(), Some("**" | "**/*"))
1543 && (pattern.owner.is_none() || owner == pattern.owner.as_deref())
1544 }) && !patterns
1545 .iter()
1546 .any(|pattern| pattern.excluded && reaches(&pattern.prefix))
1547 })
1548 }
1549}
1550
1551pub(crate) struct Coverage {
1558 resolved: Resolved,
1559 dependency_outputs: Vec<(Arc<Pattern>, Vec<Outputs>)>,
1562}
1563
1564pub(crate) fn coverage(
1566 snapshot: &Snapshot,
1567 workspace: &Workspace,
1568 graph: &TaskGraph,
1569 task: &Task,
1570 prepared: &PreparedTask,
1571 cancelled: &AtomicBool,
1572) -> Result<Coverage> {
1573 let resolved = resolve(snapshot, workspace, task, Some(prepared), cancelled)?;
1574 let selections: Vec<(String, bool)> = resolved
1575 .dependency_output_inputs()
1576 .map(|(pattern, transitive)| (pattern.to_owned(), transitive))
1577 .collect();
1578 let mut dependency_outputs = Vec::new();
1579 for (pattern, transitive) in selections {
1580 let matcher = snapshot.pattern(&pattern, false)?;
1581 let mut selected = task.dependencies.clone();
1582 if transitive {
1583 let mut pending = selected.clone();
1584 while let Some(id) = pending.pop_first() {
1585 for dependency in &graph.tasks[&id].dependencies {
1586 if selected.insert(dependency.clone()) {
1587 pending.insert(dependency.clone());
1588 }
1589 }
1590 }
1591 }
1592 let mut outputs = Vec::new();
1593 for id in selected {
1594 let declared = Outputs::new(workspace, &graph.tasks[&id])?;
1595 if declared.declared() {
1596 outputs.push(declared);
1597 }
1598 }
1599 dependency_outputs.push((matcher, outputs));
1600 }
1601 Ok(Coverage {
1602 resolved,
1603 dependency_outputs,
1604 })
1605}
1606
1607impl Coverage {
1608 fn keyed(&self, path: &str) -> bool {
1611 let inferred = &self.resolved.inferred;
1612 self.resolved.is_whole_file(path)
1613 || self.resolved.is_json_input(path)
1614 || inferred.manifests.contains_key(path)
1615 || inferred.tsconfig.as_deref() == Some(path)
1616 || inferred.settings.contains(path)
1617 || inferred.project.contains(path)
1618 || (inferred.workspace_file && path == "pnpm-workspace.yaml")
1619 || (inferred.lockfile.is_some() && path == "pnpm-lock.yaml")
1620 || self.dependency_outputs.iter().any(|(matcher, outputs)| {
1621 matcher.is_match(path) && outputs.iter().any(|outputs| outputs.matches(path))
1622 })
1623 }
1624
1625 pub(crate) fn covers(
1636 &self,
1637 snapshot: &Snapshot,
1638 root: &Path,
1639 set: &crate::recorded::PathSet,
1640 output: &dyn Fn(&str) -> bool,
1641 ) -> bool {
1642 self.uncovered(snapshot, root, set, output).next().is_none()
1643 }
1644
1645 pub(crate) fn uncovered<'a>(
1649 &'a self,
1650 snapshot: &'a Snapshot,
1651 root: &'a Path,
1652 set: &'a crate::recorded::PathSet,
1653 output: &'a dyn Fn(&str) -> bool,
1654 ) -> impl Iterator<Item = (&'a str, &'static str)> + 'a {
1655 let selection = &self.resolved.declared.selection;
1656 fn directory(path: &str) -> &str {
1657 if path == "." { "" } else { path }
1658 }
1659 let holds_keyed = move |path: &str| {
1660 let path = directory(path);
1661 if path.is_empty() {
1662 return self.resolved.whole_files().next().is_some();
1663 }
1664 under(&self.resolved.declared.files, path).next().is_some()
1665 || under(&self.resolved.inferred.files, path).next().is_some()
1666 };
1667 let selected = move |path: &str| {
1669 !snapshot.source_ignore.matches(path)
1670 && !output(path)
1671 && selection.selects(snapshot, path)
1672 };
1673 let gone = move |path: &str| {
1678 root.join(path).symlink_metadata().is_err_and(|error| {
1679 matches!(
1680 error.kind(),
1681 std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
1682 )
1683 })
1684 };
1685 let reads = set
1686 .reads
1687 .iter()
1688 .filter(move |path| !(self.keyed(path) || (gone(path) && selected(path))))
1689 .map(|path| (path.as_str(), "read"));
1690 let present = set
1691 .present
1692 .iter()
1693 .filter(move |path| {
1694 !(self.keyed(path) || (root.join(path).is_dir() && holds_keyed(path)))
1695 })
1696 .map(|path| (path.as_str(), "found"));
1697 let absent = set
1698 .absent
1699 .iter()
1700 .filter(move |path| !(self.keyed(path) || selected(path)))
1701 .map(|path| (path.as_str(), "looked for"));
1702 let listings = set
1703 .listings
1704 .iter()
1705 .filter(move |path| {
1706 let path = directory(path);
1707 !((path.is_empty() || !output(path))
1708 && selection.selects_every_entry(snapshot, path))
1709 })
1710 .map(|path| (path.as_str(), "listed"));
1711 reads.chain(present).chain(absent).chain(listings)
1712 }
1713}
1714
1715struct JsonGlob {
1717 matcher: Arc<Pattern>,
1718 prefix: String,
1719 owner: Option<String>,
1721 fields: JsonFields,
1722}
1723
1724#[derive(Default)]
1725struct FileSelection {
1726 patterns: Vec<FilePattern>,
1727 included: BTreeSet<String>,
1728 excluded: BTreeSet<String>,
1729}
1730
1731#[derive(PartialEq, Eq, PartialOrd, Ord)]
1732enum FileScope {
1733 Workspace,
1734 Project(String),
1735}
1736
1737struct Resolver<'a> {
1738 workspace: &'a Workspace,
1739 snapshot: &'a Snapshot,
1740 candidates: &'a BTreeSet<String>,
1741 selected: BTreeSet<String>,
1742 scopes: BTreeMap<FileScope, FileSelection>,
1743 values: BTreeMap<String, Value>,
1744 json_fields: BTreeMap<String, Vec<JsonFields>>,
1745 json_globs: Vec<JsonGlob>,
1746 named_stack: Vec<(String, String)>,
1747 expanded: BTreeSet<(String, String)>,
1749 external: BTreeSet<String>,
1750 ignored: BTreeMap<(String, bool), Vec<String>>,
1754 external_declared: bool,
1755 projects: BTreeSet<String>,
1758 prepared: Option<&'a PreparedTask>,
1761 cancelled: &'a AtomicBool,
1762}
1763
1764impl<'a> Resolver<'a> {
1765 fn new(
1766 snapshot: &'a Snapshot,
1767 workspace: &'a Workspace,
1768 prepared: Option<&'a PreparedTask>,
1769 cancelled: &'a AtomicBool,
1770 candidates: &'a BTreeSet<String>,
1771 ) -> Self {
1772 Self {
1773 workspace,
1774 snapshot,
1775 candidates,
1776 selected: BTreeSet::new(),
1777 scopes: BTreeMap::new(),
1778 values: BTreeMap::new(),
1779 json_fields: BTreeMap::new(),
1780 json_globs: Vec::new(),
1781 named_stack: Vec::new(),
1782 expanded: BTreeSet::new(),
1783 external: BTreeSet::new(),
1784 ignored: BTreeMap::new(),
1785 external_declared: false,
1786 projects: BTreeSet::new(),
1787 prepared,
1788 cancelled,
1789 }
1790 }
1791
1792 fn task_inputs(&mut self, task: &Task) -> Result<()> {
1793 self.projects.insert(task.project.clone());
1794 for input in declared_inputs(task) {
1795 self.input(&task.project, &input)?;
1796 }
1797 let snapshot = self.snapshot;
1800 for (project, inputs) in snapshot.servers.get(&task.id).into_iter().flatten() {
1801 for input in inputs {
1802 self.input(project, input)
1803 .with_context(|| format!("inputs of {project}, which serves this task"))?;
1804 }
1805 }
1806 Ok(())
1807 }
1808
1809 fn input(&mut self, project: &str, input: &Value) -> Result<()> {
1810 self.projects.insert(project.to_owned());
1811 if let Value::Object(object) = input
1814 && let Some(always) = object.get("always")
1815 {
1816 if always.as_bool().context("input always must be boolean")?
1819 && let Some(Value::String(name)) = object.get("input")
1820 && let Some(projects) = object.get("projects")
1821 {
1822 let patterns: Vec<String> = match projects {
1823 Value::String(pattern) => vec![pattern.clone()],
1824 other => serde_json::from_value(other.clone())
1825 .context("input projects must be a string or an array")?,
1826 };
1827 for pattern in patterns.iter().filter(|pattern| {
1828 !pattern.starts_with('!')
1829 && !matches!(pattern.as_str(), "self" | "dependencies")
1830 }) {
1831 if qk_graph::select_projects(
1832 &self.workspace.projects,
1833 std::slice::from_ref(pattern),
1834 &[],
1835 )?
1836 .is_empty()
1837 {
1838 bail!(
1839 "the input {{\"input\": \"{name}\", \"projects\"}} with \"always\": true selects \"{pattern}\", which matches no project"
1840 );
1841 }
1842 }
1843 }
1844 let mut object = object.clone();
1845 object.remove("always");
1846 return self.input(project, &Value::Object(object));
1847 }
1848 match input {
1849 Value::String(name)
1853 if name.starts_with("^{projectRoot}") || name.starts_with("^{workspaceRoot}") =>
1854 {
1855 self.dependency_fileset(project, &name[1..])?;
1856 }
1857 Value::String(name) if name.starts_with('^') => {
1858 self.dependencies_named(project, &name[1..])?;
1859 }
1860 Value::Object(object)
1862 if object.contains_key("input") && !object.contains_key("fileset") =>
1863 {
1864 let name = object["input"]
1865 .as_str()
1866 .context("input must name a named input")?;
1867 let projects = object.get("projects");
1868 let dependencies = object.get("dependencies") == Some(&json!(true))
1869 || projects == Some(&json!("dependencies"));
1870 let listed = projects.filter(|projects| {
1871 **projects != json!("self") && **projects != json!("dependencies")
1872 });
1873 if (dependencies || listed.is_some()) && !self.named_stack.is_empty() {
1874 bail!("named inputs can only refer to named inputs of their own project");
1875 }
1876 if let Some(listed) = listed {
1877 let patterns: Vec<String> = match listed {
1878 Value::String(pattern) => vec![pattern.clone()],
1879 other => serde_json::from_value(other.clone())
1880 .context("input projects must be a string or an array")?,
1881 };
1882 for selected in qk_config::find_matching_projects(
1885 &patterns,
1886 &self.workspace.projects,
1887 qk_config::Reader::Native,
1888 ) {
1889 if dependencies {
1892 self.dependencies_named(&selected, name)?;
1893 } else if self.expanded.insert((selected.clone(), name.to_owned())) {
1894 self.named(&selected, name)?;
1895 }
1896 }
1897 } else if dependencies {
1898 self.dependencies_named(project, name)?;
1899 } else {
1900 self.named(project, name)?;
1901 }
1902 }
1903 Value::Object(object) if object.contains_key("fileset") => {
1904 if let Some(key) = object.keys().find(|key| {
1905 !matches!(key.as_str(), "fileset" | "dependencies" | "includeIgnored")
1906 }) {
1907 bail!("unsupported fileset input field {key:?}");
1908 }
1909 let flag = |name: &str| {
1910 object
1911 .get(name)
1912 .map(|value| {
1913 value
1914 .as_bool()
1915 .with_context(|| format!("{name} must be boolean"))
1916 })
1917 .transpose()
1918 .map(Option::unwrap_or_default)
1919 };
1920 let (dependencies, include_ignored) =
1921 (flag("dependencies")?, flag("includeIgnored")?);
1922 let pattern = object["fileset"]
1923 .as_str()
1924 .context("fileset input must be a glob")?
1925 .to_owned();
1926 validate_fileset(&pattern)?;
1927 match (dependencies, include_ignored) {
1928 (false, false) => self.fileset(project, &pattern)?,
1929 (true, false) => self.dependency_fileset(project, &pattern)?,
1930 (false, true) => self.ignored_fileset(project, &pattern, false),
1931 (true, true) => {
1932 if !self.named_stack.is_empty() {
1933 bail!(
1934 "named inputs can only refer to named inputs of their own project"
1935 );
1936 }
1937 for dependency in self.dependency_closure(project) {
1938 self.ignored_fileset(&dependency, &pattern, true);
1939 }
1940 }
1941 }
1942 }
1943 Value::Object(object)
1944 if object.len() == 1 && object.contains_key("workingDirectory") =>
1945 {
1946 let mode = object["workingDirectory"].as_str().unwrap_or_default();
1947 let directory = std::env::current_dir()?;
1948 let value = match mode {
1949 "absolute" => directory.to_string_lossy().into_owned(),
1950 "relative" => {
1951 let root = self.workspace.root.canonicalize()?;
1952 let directory = directory.canonicalize()?;
1953 paths::relative(&root, &directory)
1954 .unwrap_or_else(|_| directory.to_string_lossy().into_owned())
1955 }
1956 _ => bail!("workingDirectory must be \"relative\" or \"absolute\""),
1957 };
1958 self.values
1959 .insert(format!("workingDirectory:{mode}"), json!(value));
1960 }
1961 Value::Object(object) if object.contains_key("json") => {
1962 self.json(project, object)?;
1963 }
1964 Value::String(name) if self.has_named(project, name) => self.named(project, name)?,
1967 Value::String(pattern) => {
1968 validate_fileset(pattern)?;
1969 self.fileset(project, pattern)?
1970 }
1971 Value::Object(object) if object.len() == 1 && object.contains_key("env") => {
1972 let name = object["env"]
1973 .as_str()
1974 .context("env input must name a variable")?;
1975 let Some(prepared) = self.prepared else {
1976 self.values.insert(format!("env:{name}"), Value::Null);
1977 return Ok(());
1978 };
1979 let value = prepared
1980 .env
1981 .get(std::ffi::OsStr::new(name))
1982 .map(|value| {
1983 value
1984 .to_str()
1985 .map(str::to_owned)
1986 .context("declared env input must be UTF-8")
1987 })
1988 .transpose()?;
1989 self.values.insert(format!("env:{name}"), json!(value));
1990 }
1991 Value::Object(object) if object.len() == 1 && object.contains_key("runtime") => {
1992 let command = object["runtime"]
1993 .as_str()
1994 .context("runtime input must be a command")?;
1995 let Some(prepared) = self.prepared else {
1996 self.values
1997 .insert(format!("runtime:{command}"), Value::Null);
1998 return Ok(());
1999 };
2000 let memo = RuntimeKey {
2001 command: command.into(),
2002 environment: prepared.runtime_env.clone(),
2003 };
2004 let slot = self
2005 .snapshot
2006 .runtime
2007 .lock()
2008 .unwrap()
2009 .entry(memo)
2010 .or_default()
2011 .clone();
2012 let mut saved = slot.lock().unwrap();
2013 if let Some(value) = saved.as_ref() {
2014 self.values
2015 .insert(format!("runtime:{command}"), value.clone());
2016 return Ok(());
2017 }
2018 let _profile = crate::profile::span(command, "runtime");
2019 let mut prepared = prepared.clone();
2020 prepared.env = prepared.runtime_env.clone();
2021 prepared.execution.clear();
2022 prepared.node_path = None;
2023 prepared.commands = vec![command.into()];
2024 prepared.cwd = self.workspace.root.clone();
2025 prepared.parallel = false;
2026 prepared.pseudo_terminal = false;
2028 let log = tempfile::NamedTempFile::new()?;
2029 let capture = Capture::new(Some(log.as_file().try_clone()?), Display::Hidden);
2030 if execute_captured(&prepared, self.cancelled, Some(&capture))? != Outcome::Success
2031 {
2032 bail!("runtime input did not succeed");
2033 }
2034 let mut stdout = blake3::Hasher::new();
2035 let mut stderr = blake3::Hasher::new();
2036 read_capture(File::open(log.path())?, |error, bytes| {
2037 if error {
2038 stderr.update(bytes);
2039 } else {
2040 stdout.update(bytes);
2041 }
2042 Ok(())
2043 })?;
2044 let value = json!([
2045 stdout.finalize().to_hex().to_string(),
2046 stderr.finalize().to_hex().to_string()
2047 ]);
2048 *saved = Some(value.clone());
2049 self.values.insert(format!("runtime:{command}"), value);
2050 }
2051 Value::Object(object) if object.contains_key("dependentTasksOutputFiles") => {
2052 if object
2053 .keys()
2054 .any(|key| !matches!(key.as_str(), "dependentTasksOutputFiles" | "transitive"))
2055 {
2056 bail!("unsupported output input field");
2057 }
2058 let pattern = object["dependentTasksOutputFiles"]
2059 .as_str()
2060 .context("dependentTasksOutputFiles must be a glob")?;
2061 self.snapshot.pattern(pattern, false)?;
2062 let transitive = object
2063 .get("transitive")
2064 .map(|value| value.as_bool().context("transitive must be boolean"))
2065 .transpose()?
2066 .unwrap_or(false);
2067 let value = self
2071 .values
2072 .entry(format!("dependentTasksOutputFiles:{pattern}"))
2073 .or_insert(json!(false));
2074 if transitive {
2075 *value = json!(true);
2076 }
2077 }
2078 Value::Object(object)
2079 if object.len() == 1 && object.contains_key("externalDependencies") =>
2080 {
2081 let names: Vec<String> =
2082 serde_json::from_value(object["externalDependencies"].clone())?;
2083 if let Some(name) = names
2085 .iter()
2086 .find(|name| self.workspace.projects.contains_key(*name))
2087 {
2088 if self.snapshot.projects.dependencies[project]
2089 .iter()
2090 .any(|edge| edge.target == *name)
2091 {
2092 bail!(
2093 "the externalDependency {name:?} of {project} is not an external node and is already a dependency; remove it from the externalDependencies inputs"
2094 );
2095 }
2096 bail!(
2097 "the externalDependency {name:?} of {project} is not an external node; if it is a dependency, add it to the project's implicitDependencies"
2098 );
2099 }
2100 self.external.extend(names);
2101 self.external_declared = true;
2102 }
2103 _ => bail!("unsupported task input declaration"),
2104 }
2105 Ok(())
2106 }
2107
2108 fn fileset(&mut self, project: &str, pattern: &str) -> Result<()> {
2109 self.projects.insert(project.to_owned());
2110 let (exclude, pattern) = pattern
2111 .strip_prefix('!')
2112 .map(|pattern| (true, pattern))
2113 .unwrap_or((false, pattern));
2114 let (scope, owner) = if pattern.starts_with("{projectRoot}/") {
2120 (
2121 FileScope::Project(project.to_owned()),
2122 Some(project.to_owned()),
2123 )
2124 } else if pattern.starts_with("{workspaceRoot}/") {
2125 (FileScope::Workspace, None)
2126 } else {
2127 return Ok(());
2128 };
2129 let pattern = paths::expand_glob(self.workspace, project, pattern)?;
2130 let matcher = self.snapshot.fileset_pattern(&pattern, exclude)?;
2131 let (prefix, rest) = crate::ignored::partition(pattern.trim_end_matches('/'));
2132 let rest = rest.map(str::to_owned);
2133 let candidates: Box<dyn Iterator<Item = &String>> = if prefix.is_empty() {
2134 Box::new(self.candidates.iter())
2135 } else {
2136 Box::new(under(self.candidates, &prefix))
2137 };
2138 let snapshot = self.snapshot;
2139 let selection = self.scopes.entry(scope).or_default();
2140 let selected = if exclude {
2141 &mut selection.excluded
2142 } else {
2143 &mut selection.included
2144 };
2145 selected.extend(
2146 candidates
2147 .filter(|file| matcher.is_match(file))
2148 .filter(|file| owner.is_none() || snapshot.owner(file) == owner.as_deref())
2149 .cloned(),
2150 );
2151 selection.patterns.push(FilePattern {
2152 matcher,
2153 prefix,
2154 rest,
2155 excluded: exclude,
2156 owner,
2157 });
2158 Ok(())
2159 }
2160
2161 fn ignored_fileset(&mut self, project: &str, pattern: &str, dependencies: bool) {
2164 let root = &self.workspace.projects[project].root;
2167 let resolved = if root == "." {
2168 pattern.replace("{projectRoot}/", "")
2169 } else {
2170 pattern.replace("{projectRoot}", &crate::glob::escape(root))
2171 }
2172 .replace("{projectName}", &crate::glob::escape(project));
2173 let resolved = match resolved.strip_prefix("!{workspaceRoot}/") {
2174 Some(rest) => format!("!{rest}"),
2175 None => resolved
2176 .strip_prefix("{workspaceRoot}/")
2177 .map(str::to_owned)
2178 .unwrap_or(resolved),
2179 };
2180 self.ignored
2181 .entry((project.to_owned(), dependencies))
2182 .or_default()
2183 .push(resolved);
2184 }
2185
2186 fn dependency_fileset(&mut self, project: &str, pattern: &str) -> Result<()> {
2188 if !self.named_stack.is_empty() {
2189 bail!("named inputs can only refer to named inputs of their own project");
2190 }
2191 for dependency in self.dependency_closure(project) {
2192 self.fileset(&dependency, pattern)?;
2193 }
2194 Ok(())
2195 }
2196
2197 fn has_named(&self, project: &str, name: &str) -> bool {
2199 name == "default"
2200 || self.workspace.projects[project]
2201 .named_inputs
2202 .contains_key(name)
2203 }
2204
2205 fn dependency_closure(&self, project: &str) -> BTreeSet<String> {
2207 let mut closure = BTreeSet::new();
2208 let mut pending = vec![project.to_owned()];
2209 while let Some(current) = pending.pop() {
2210 for edge in &self.snapshot.projects.dependencies[¤t] {
2211 if edge.target != project && closure.insert(edge.target.clone()) {
2212 pending.push(edge.target.clone());
2213 }
2214 }
2215 }
2216 closure
2217 }
2218
2219 fn dependencies_named(&mut self, project: &str, name: &str) -> Result<()> {
2221 for dependency in self.dependency_closure(project) {
2222 if self.expanded.insert((dependency.clone(), name.to_owned())) {
2223 self.named(&dependency, name)?;
2224 }
2225 }
2226 Ok(())
2227 }
2228
2229 fn json(&mut self, project: &str, object: &serde_json::Map<String, Value>) -> Result<()> {
2233 for key in object.keys() {
2234 if !matches!(key.as_str(), "json" | "fields" | "excludeFields") {
2235 bail!("unsupported json input field {key:?}");
2236 }
2237 }
2238 let declared = object["json"]
2239 .as_str()
2240 .context("json input must name a file")?;
2241 validate_fileset(declared)?;
2242 let path = paths::expand_glob(self.workspace, project, declared)?;
2243 let list = |name: &str| -> Result<Option<Vec<String>>> {
2244 object
2245 .get(name)
2246 .map(|value| {
2247 serde_json::from_value(value.clone())
2248 .with_context(|| format!("json input {name} must be an array of strings"))
2249 })
2250 .transpose()
2251 };
2252 let fields = JsonFields {
2253 fields: list("fields")?,
2254 exclude: list("excludeFields")?.unwrap_or_default(),
2255 };
2256 if crate::glob::names_a_path(&path) {
2257 let path = crate::glob::literal_path(&path);
2258 paths::validate_path(&path)?;
2259 return self.json_file(path, &fields);
2260 }
2261 let owner = if declared.starts_with("{projectRoot}") {
2264 Some(project)
2265 } else if declared.starts_with("{workspaceRoot}/") {
2266 None
2267 } else {
2268 return Ok(());
2269 };
2270 let matcher = self.snapshot.pattern(&path, false)?;
2271 let snapshot = self.snapshot;
2272 let matched: Vec<String> = self
2273 .candidates
2274 .iter()
2275 .filter(|file| matcher.is_match(file))
2276 .filter(|file| owner.is_none() || snapshot.owner(file) == owner)
2277 .cloned()
2278 .collect();
2279 for file in matched {
2280 self.json_file(file, &fields)?;
2281 }
2282 self.json_globs.push(JsonGlob {
2283 matcher,
2284 prefix: crate::ignored::partition(&path).0,
2285 owner: owner.map(str::to_owned),
2286 fields,
2287 });
2288 Ok(())
2289 }
2290
2291 fn json_file(&mut self, path: String, fields: &JsonFields) -> Result<()> {
2295 let text = match std::fs::read_to_string(self.workspace.root.join(&path)) {
2296 Ok(text) => Some(text),
2297 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
2298 Err(error) => return Err(error.into()),
2299 };
2300 let mut selections = self.json_fields.get(&path).cloned().unwrap_or_default();
2301 if !selections.contains(fields) {
2302 selections.push(fields.clone());
2303 selections.sort();
2305 }
2306 let mut values = Vec::new();
2307 for selection in &selections {
2308 values.push(match &text {
2309 Some(text) => match json_projection(&path, text, selection) {
2310 Some(value) => value,
2311 None => {
2315 self.selected.insert(path);
2316 return Ok(());
2317 }
2318 },
2319 None => Value::Null,
2320 });
2321 }
2322 let value = if values.len() == 1 {
2324 values.remove(0)
2325 } else {
2326 Value::Array(values)
2327 };
2328 self.values.insert(format!("json:{path}"), value);
2329 self.json_fields.insert(path, selections);
2330 Ok(())
2331 }
2332
2333 fn named(&mut self, project: &str, name: &str) -> Result<()> {
2334 let identity = (project.to_owned(), name.to_owned());
2335 if self.named_stack.contains(&identity) {
2336 bail!("named input cycle at {project}:{name}");
2337 }
2338 self.named_stack.push(identity);
2339 let inputs = self.workspace.projects[project]
2340 .named_inputs
2341 .get(name)
2342 .cloned()
2343 .or_else(|| (name == "default").then(|| vec![json!("{projectRoot}/**/*")]))
2344 .with_context(|| format!("unknown named input {name:?}"))?;
2345 for input in &inputs {
2346 self.input(project, input)?;
2347 }
2348 self.named_stack.pop();
2349 Ok(())
2350 }
2351}
2352
2353fn validate_fileset(fileset: &str) -> Result<()> {
2356 let body = fileset.strip_prefix('!').unwrap_or(fileset);
2357 if !body.starts_with("{projectRoot}") && !body.starts_with("{workspaceRoot}") {
2358 bail!(
2359 "{fileset:?} is an invalid fileset: filesets have to start with either {{workspaceRoot}} or {{projectRoot}}, and a named input must be defined"
2360 );
2361 }
2362 Ok(())
2363}
2364
2365fn declared_inputs(task: &Task) -> Vec<Value> {
2367 task.definition
2368 .inputs
2369 .clone()
2370 .unwrap_or_else(|| vec![json!("default"), json!("^default")])
2371}
2372
2373fn servers(graph: &TaskGraph, id: &str) -> Vec<(String, Vec<Value>)> {
2379 let mut servers = Vec::new();
2380 let mut visited = BTreeSet::from([id]);
2381 let mut pending = std::collections::VecDeque::from([id]);
2382 while let Some(current) = pending.pop_front() {
2383 let task = &graph.tasks[current];
2384 for dependency in &task.dependencies {
2385 let Some(server) = graph.tasks.get(dependency) else {
2386 continue;
2387 };
2388 if server.definition.continuous != Some(true)
2389 || !task.edge(dependency).inputs
2390 || !visited.insert(dependency)
2391 {
2392 continue;
2393 }
2394 servers.push((server.project.clone(), declared_inputs(server)));
2395 pending.push_back(dependency);
2396 }
2397 }
2398 servers
2399}
2400
2401pub fn resolve(
2402 snapshot: &Snapshot,
2403 workspace: &Workspace,
2404 task: &Task,
2405 prepared: Option<&PreparedTask>,
2406 cancelled: &AtomicBool,
2407) -> Result<Resolved> {
2408 resolve_candidates(
2409 snapshot,
2410 workspace,
2411 task,
2412 prepared,
2413 cancelled,
2414 &snapshot.files,
2415 &snapshot.source_ignore,
2416 )
2417}
2418
2419fn resolve_candidates(
2420 snapshot: &Snapshot,
2421 workspace: &Workspace,
2422 task: &Task,
2423 prepared: Option<&PreparedTask>,
2424 cancelled: &AtomicBool,
2425 candidates: &BTreeSet<String>,
2426 source_ignore: &SourceIgnore,
2427) -> Result<Resolved> {
2428 let mut resolver = Resolver::new(snapshot, workspace, prepared, cancelled, candidates);
2429 resolver.task_inputs(task)?;
2430 let mut upstream = BTreeSet::new();
2434 let mut pending: Vec<_> = task.dependencies.iter().cloned().collect();
2435 while let Some(id) = pending.pop() {
2436 if upstream.insert(id.clone())
2437 && let Some(dependencies) = snapshot.task_dependencies.get(&id)
2438 {
2439 pending.extend(dependencies.iter().cloned());
2440 }
2441 }
2442 let own_outputs = Outputs::new(workspace, task)
2443 .ok()
2444 .filter(Outputs::is_explicit);
2445 let mut producers = BTreeSet::new();
2446 for (producer, declared) in &snapshot.generated_tasks {
2447 if producer.id == task.id || !upstream.contains(&producer.id) {
2448 continue;
2449 }
2450 let overlaps = |a: &str, b: &str| {
2451 a.is_empty()
2452 || a == b
2453 || a.strip_prefix(b).is_some_and(|rest| rest.starts_with('/'))
2454 || b.strip_prefix(a).is_some_and(|rest| rest.starts_with('/'))
2455 };
2456 let relevant = resolver
2457 .scopes
2458 .values()
2459 .flat_map(|scope| &scope.patterns)
2460 .filter(|pattern| !pattern.excluded)
2461 .map(|pattern| &pattern.prefix)
2462 .chain(resolver.json_globs.iter().map(|glob| &glob.prefix))
2463 .any(|prefix| {
2464 declared.anchors().any(|anchor| {
2465 overlaps(prefix, anchor)
2466 && (!source_ignore.matches_directory(anchor)
2467 || !workspace.root.join(anchor).is_dir())
2468 })
2469 });
2470 if !relevant {
2471 continue;
2472 }
2473 producers.insert(producer.id.clone());
2474 let outputs = snapshot.dependency_outputs(workspace, producer)?;
2475 let mut json = Vec::new();
2476 for path in &outputs.files {
2477 if source_ignore.matches(path)
2478 || own_outputs.as_ref().is_some_and(|own| own.matches(path))
2479 {
2480 continue;
2481 }
2482 json.extend(
2483 resolver
2484 .json_globs
2485 .iter()
2486 .filter(|glob| {
2487 glob.matcher.is_match(path)
2488 && (glob.owner.is_none()
2489 || snapshot.owner(path) == glob.owner.as_deref())
2490 })
2491 .map(|glob| (path.clone(), glob.fields.clone())),
2492 );
2493 for selection in resolver.scopes.values_mut() {
2494 for pattern in &selection.patterns {
2495 if pattern.matcher.is_match(path)
2496 && (pattern.owner.is_none()
2497 || snapshot.owner(path) == pattern.owner.as_deref())
2498 {
2499 if pattern.excluded {
2500 selection.excluded.insert(path.clone());
2501 } else {
2502 selection.included.insert(path.clone());
2503 }
2504 }
2505 }
2506 }
2507 }
2508 for (path, fields) in json {
2509 resolver.json_file(path, &fields)?;
2510 }
2511 }
2512 for selection in resolver.scopes.values() {
2515 resolver.selected.extend(
2516 selection
2517 .included
2518 .difference(&selection.excluded)
2519 .filter(|path| {
2520 snapshot.extra_candidates.contains(*path)
2521 || own_outputs.as_ref().is_none_or(|own| !own.matches(path))
2522 })
2523 .cloned(),
2524 );
2525 }
2526 let mut ignored = BTreeSet::new();
2530 for ((project, _), globs) in &resolver.ignored {
2531 let group = Group::new(
2532 project,
2533 globs,
2534 workspace
2535 .projects
2536 .values()
2537 .map(|project| project.root.as_str()),
2538 )?;
2539 let found = group.expand(&workspace.root)?;
2540 ignored.extend(
2541 found
2542 .into_iter()
2543 .chain(
2544 snapshot
2545 .extra_candidates
2546 .iter()
2547 .filter(|path| group.matches(path))
2548 .cloned(),
2549 )
2550 .filter(|path| own_outputs.as_ref().is_none_or(|own| !own.matches(path)))
2553 .filter(|path| !snapshot.scratch.iter().any(|scratch| scratch.matches(path))),
2554 );
2555 for (producer, declared) in &snapshot.generated_tasks {
2556 if producer.id != task.id
2557 && upstream.contains(&producer.id)
2558 && declared.declared()
2559 && reads_outputs(&group, declared)
2560 {
2561 producers.insert(producer.id.clone());
2562 }
2563 }
2564 }
2565 resolver.selected.extend(ignored.iter().cloned());
2566 let mut declared = Declared {
2567 files: resolver.selected,
2568 ignored,
2569 values: resolver.values,
2570 json_fields: resolver.json_fields,
2571 external: resolver.external,
2572 external_declared: resolver.external_declared,
2573 projects: resolver.projects,
2574 producers,
2575 selection: Selection {
2576 scopes: resolver
2577 .scopes
2578 .into_values()
2579 .map(|selection| selection.patterns)
2580 .collect(),
2581 },
2582 };
2583 let reads: Vec<bool> = declared
2586 .values
2587 .iter()
2588 .filter(|(name, _)| name.starts_with("dependentTasksOutputFiles:"))
2589 .map(|(_, transitive)| transitive == &Value::Bool(true))
2590 .collect();
2591 if !reads.is_empty() {
2592 let transitive = reads.contains(&true);
2593 for (producer, outputs) in &snapshot.generated_tasks {
2594 let read = if transitive {
2595 upstream.contains(&producer.id)
2596 } else {
2597 task.dependencies.contains(&producer.id)
2598 };
2599 if read && outputs.declared() {
2600 declared.producers.insert(producer.id.clone());
2601 }
2602 }
2603 }
2604 let mut inferred = infer(snapshot, workspace, task)?;
2605 if declared.external_declared
2609 && let Some(importers) = &mut inferred.lockfile
2610 {
2611 let root = workspace.projects[&task.project].root == ".";
2612 if !root {
2613 importers.remove(".");
2614 }
2615 }
2616 Ok(join(declared, inferred))
2617}
2618
2619fn reads_outputs(group: &Group, outputs: &Outputs) -> bool {
2625 let holds = |outer: &str, inner: &str| {
2626 outer.is_empty()
2627 || inner == outer
2628 || inner
2629 .strip_prefix(outer)
2630 .is_some_and(|rest| rest.starts_with('/'))
2631 };
2632 group.reads().any(|(root, extension)| {
2633 outputs.globs().iter().any(|output| {
2634 let (prefix, _) = crate::ignored::partition(output);
2635 (holds(root, &prefix) || holds(&prefix, root))
2636 && !group
2637 .skipped_below(root)
2638 .iter()
2639 .any(|project| holds(project, &prefix))
2640 && match (extension, crate::ignored::extension(output)) {
2641 (Some(read), Some(written)) => read == written,
2642 _ => true,
2643 }
2644 })
2645 })
2646}
2647
2648fn infer(snapshot: &Snapshot, workspace: &Workspace, task: &Task) -> Result<Inferred> {
2650 let readable = snapshot
2651 .installed(&workspace.root, &task.id)?
2652 .is_some_and(|installed| installed.lockfile.is_some());
2653 let exists = |path: &str| {
2654 workspace.root.join(path).is_file() || snapshot.extra_candidates.contains(path)
2655 };
2656 let mut inferred = Inferred {
2657 lockfile_readable: readable,
2658 ..Inferred::default()
2659 };
2660 inferred.tsconfig = crate::inputs::ROOT_TSCONFIG_FILES
2667 .into_iter()
2668 .find(|name| exists(name))
2669 .map(str::to_owned);
2670 for path in ["nx.json", qk_config::LOCAL_WORKSPACE] {
2674 if exists(path) {
2675 inferred.settings.insert(path.into());
2676 }
2677 }
2678 if let Some(extended) = &workspace.extended {
2679 inferred.settings.insert(extended.clone());
2680 }
2681 for path in [
2682 ".gitignore",
2683 ".nxignore",
2684 "package-lock.json",
2685 "yarn.lock",
2686 "bun.lock",
2687 ] {
2688 if exists(path) {
2689 inferred.files.insert(path.into());
2690 }
2691 }
2692 if exists("pnpm-workspace.yaml") {
2696 if readable {
2697 inferred.workspace_file = true;
2698 } else {
2699 inferred.files.insert("pnpm-workspace.yaml".into());
2700 }
2701 }
2702 if exists("pnpm-lock.yaml") && !readable {
2703 inferred.files.insert("pnpm-lock.yaml".into());
2704 }
2705 let mut packages = BTreeSet::from([task.project.clone()]);
2710 let mut pending = packages.clone();
2711 while let Some(project) = pending.pop_first() {
2712 for edge in &snapshot.projects.dependencies[&project] {
2713 if packages.insert(edge.target.clone()) {
2714 pending.insert(edge.target.clone());
2715 }
2716 }
2717 }
2718 for project in &packages {
2719 let own = *project == task.project;
2720 for name in ["project.json", qk_config::LOCAL_OVERRIDES, "package.json"] {
2721 if !own && name != "package.json" {
2722 continue;
2723 }
2724 let path = Path::new(&workspace.projects[project].root).join(name);
2725 let path = path.strip_prefix(".").unwrap_or(&path).to_owned();
2726 if !workspace.root.join(&path).is_file() {
2727 continue;
2728 }
2729 let path = path
2730 .to_str()
2731 .context("project path must be UTF-8")?
2732 .replace('\\', "/");
2733 if own && name != "package.json" {
2738 inferred.project.insert(path);
2739 continue;
2740 }
2741 if own {
2742 inferred.files.insert(path);
2743 continue;
2744 }
2745 let text = std::fs::read_to_string(workspace.root.join(&path))
2746 .with_context(|| format!("cannot read {path}"))?;
2747 match manifest_for_dependents(&text, readable) {
2748 Some(value) => {
2749 inferred.manifests.insert(path, value);
2750 }
2751 None => {
2752 inferred.files.insert(path);
2753 }
2754 }
2755 }
2756 }
2757 if readable {
2758 inferred.lockfile = Some(
2761 std::iter::once(".".to_owned())
2762 .chain(
2763 packages
2764 .iter()
2765 .map(|project| workspace.projects[project].root.clone()),
2766 )
2767 .collect(),
2768 );
2769 }
2770 Ok(inferred)
2771}
2772
2773fn join(declared: Declared, mut inferred: Inferred) -> Resolved {
2776 let mut covered = BTreeSet::new();
2777 inferred.files.retain(|path| {
2778 let keep = !declared.files.contains(path);
2779 if !keep {
2780 covered.insert(path.clone());
2781 }
2782 keep
2783 });
2784 inferred.manifests.retain(|path, _| {
2785 let keep = !declared.files.contains(path);
2786 if !keep {
2787 covered.insert(path.clone());
2788 }
2789 keep
2790 });
2791 if inferred.lockfile.is_some() && declared.files.contains("pnpm-lock.yaml") {
2792 inferred.lockfile = None;
2793 covered.insert("pnpm-lock.yaml".into());
2794 }
2795 if inferred.workspace_file && declared.files.contains("pnpm-workspace.yaml") {
2796 inferred.workspace_file = false;
2797 covered.insert("pnpm-workspace.yaml".into());
2798 }
2799 if let Some(tsconfig) = inferred
2800 .tsconfig
2801 .take_if(|path| declared.files.contains(path))
2802 {
2803 covered.insert(tsconfig);
2804 }
2805 inferred.covered = covered;
2806 Resolved { declared, inferred }
2807}
2808
2809pub fn inputs(
2812 snapshot: &Snapshot,
2813 workspace: &Workspace,
2814 task: &Task,
2815 prepared: &PreparedTask,
2816 dependencies: &BTreeMap<String, String>,
2817 cancelled: &AtomicBool,
2818) -> Result<Value> {
2819 let prefixes = snapshot.membership_prefixes(workspace, task)?;
2820 let candidates = snapshot.current_files_for(&workspace.root, Some(&prefixes), &task.id)?;
2821 let Resolved { declared, inferred } = resolve_candidates(
2822 snapshot,
2823 workspace,
2824 task,
2825 Some(prepared),
2826 cancelled,
2827 &candidates.files,
2828 &candidates.source_ignore,
2829 )?;
2830 let file_values = |paths: &BTreeSet<String>| {
2831 paths
2832 .iter()
2833 .map(|path| {
2834 let value = snapshot
2835 .file_value(&workspace.root, path)
2836 .with_context(|| format!("cannot read input {path}"))?;
2837 Ok((path.clone(), value))
2838 })
2839 .collect::<Result<BTreeMap<_, _>>>()
2840 };
2841 let mut files = file_values(&(&declared.files - &declared.ignored))?;
2842 for path in &declared.ignored {
2843 if let Some(value) = snapshot
2844 .ignored_value(&workspace.root, path)
2845 .with_context(|| format!("cannot read input {path}"))?
2846 {
2847 files.insert(path.clone(), value);
2848 }
2849 }
2850 let mut values = declared.values;
2851 if let Some(environment) = &prepared.environment {
2854 values.insert("environment".into(), environment.keyed());
2855 }
2856 let mut configuration = json!({
2857 "files": file_values(&inferred.files)?,
2858 "settings": workspace.config.keyed_settings(),
2859 "project": task_configuration(workspace, task),
2860 });
2861 let installed = snapshot.installed(&workspace.root, &task.id)?;
2862 if let (Some(importers), Some(installed)) = (&inferred.lockfile, &installed)
2863 && let Some(lockfile) = &installed.lockfile
2864 {
2865 let importers: BTreeMap<_, _> = importers
2866 .iter()
2867 .filter(|importer| lockfile.has_importer(importer))
2868 .map(|importer| {
2869 let digest = installed.digest(&format!("importer:{importer}"), || {
2870 lockfile.installed(importer).unwrap_or_default()
2871 });
2872 (importer.clone(), digest)
2873 })
2874 .collect();
2875 configuration["lockfile"] = json!({
2876 "global": installed.digest("global", || BTreeSet::from([lockfile.global().to_string()])),
2877 "importers": importers,
2878 });
2879 if !declared.external.is_empty() {
2881 let external: BTreeMap<_, _> = declared
2882 .external
2883 .iter()
2884 .map(|name| {
2885 let digest =
2886 installed.digest(&format!("package:{name}"), || lockfile.package(name));
2887 if digest == blake3::hash(b"").to_hex().as_str() {
2889 bail!(
2890 "the externalDependency {name:?} of {} could not be found",
2891 task.id
2892 );
2893 }
2894 Ok((name.clone(), digest))
2895 })
2896 .collect::<Result<_>>()?;
2897 values.insert("externalDependencies".into(), json!(external));
2898 }
2899 }
2900 if inferred.workspace_file {
2901 configuration["workspaceFile"] = match workspace_without_resolution(&workspace.root) {
2902 Some(value) => value,
2903 None => snapshot.file_value(&workspace.root, "pnpm-workspace.yaml")?,
2904 };
2905 }
2906 if !inferred.manifests.is_empty() {
2907 configuration["manifests"] = json!(inferred.manifests);
2908 }
2909 if let Some(path) = &inferred.tsconfig {
2910 let parsed = std::fs::read_to_string(workspace.root.join(path))
2911 .ok()
2912 .and_then(|text| TsConfig::parse(&text));
2913 configuration["tsconfig"] = match parsed {
2914 Some(tsconfig) if crate::inputs::selectively_hash_tsconfig(workspace) => json!({
2915 "rest": tsconfig.rest,
2916 "paths": tsconfig.paths_of(&declared.projects, |target| snapshot.owner(target)),
2917 }),
2918 Some(tsconfig) => json!({ "rest": tsconfig.rest }),
2919 None => snapshot.file_value(&workspace.root, path)?,
2920 };
2921 }
2922 let definition = keyed_definition(task);
2923 let hash = json!({
2924 "schema":"qk-local-v4", "qk":env!("CARGO_PKG_VERSION"),
2925 "platform":[std::env::consts::OS, std::env::consts::ARCH], "workspace":snapshot.workspace_prefix,
2926 "id":task.id, "args":task.args, "definition":definition, "packageManager":workspace.package_manager,
2927 "files":files, "values":values, "inferred":configuration, "dependencies":dependencies,
2928 });
2929 Ok(hash)
2930}
2931
2932pub fn keyed_definition(task: &Task) -> qk_config::Target {
2936 let mut definition = task.definition.clone();
2937 definition.extra.remove("qk:warm");
2938 definition.extra.remove("qk:threads");
2939 definition.extra.remove("metadata");
2940 definition.extra.remove("qk:ultracache");
2942 definition
2943}
2944
2945pub fn task_configuration(workspace: &Workspace, task: &Task) -> Value {
2951 let project = &workspace.projects[&task.project];
2952 let mut value = json!({ "settings": project.keyed_settings() });
2953 if let Ok((outputs, _)) = paths::resolved_outputs(workspace, task) {
2954 value["outputs"] = json!(outputs);
2955 }
2956 if task.args.iter().any(|arg| arg.contains("{project")) {
2957 value["configuration"] = serde_json::to_value(project).unwrap_or_default();
2958 }
2959 value
2960}
2961
2962pub fn key(inputs: &Value) -> Result<String> {
2966 Ok(blake3::hash(&serde_json::to_vec(&sorted(inputs))?)
2967 .to_hex()
2968 .to_string())
2969}
2970
2971fn sorted(value: &Value) -> Value {
2973 match value {
2974 Value::Object(object) => {
2975 let mut fields: Vec<_> = object.iter().collect();
2976 fields.sort_unstable_by_key(|(name, _)| *name);
2977 Value::Object(
2978 fields
2979 .into_iter()
2980 .map(|(name, value)| (name.clone(), sorted(value)))
2981 .collect(),
2982 )
2983 }
2984 Value::Array(items) => Value::Array(items.iter().map(sorted).collect()),
2985 other => other.clone(),
2986 }
2987}
2988
2989pub fn fingerprint(
2990 snapshot: &Snapshot,
2991 workspace: &Workspace,
2992 task: &Task,
2993 prepared: &PreparedTask,
2994 dependencies: &BTreeMap<String, String>,
2995 cancelled: &AtomicBool,
2996) -> Result<String> {
2997 key(&inputs(
2998 snapshot,
2999 workspace,
3000 task,
3001 prepared,
3002 dependencies,
3003 cancelled,
3004 )?)
3005}
3006
3007#[allow(clippy::too_many_arguments)]
3012pub(crate) fn weak_key(
3013 snapshot: &Snapshot,
3014 workspace: &Workspace,
3015 graph: &TaskGraph,
3016 task: &Task,
3017 prepared: &PreparedTask,
3018 inputs: &Value,
3019 ignore_reads: &[String],
3020 ignored_within: &[(&str, bool)],
3021 cancelled: &AtomicBool,
3022) -> Result<String> {
3023 let mut weak = inputs.clone();
3024 let object = weak
3025 .as_object_mut()
3026 .context("a key's inputs are an object")?;
3027 object.remove("files");
3028 object.remove("dependencies");
3029 if let Some(Value::Object(values)) = object.get_mut("values") {
3030 values.retain(|name, _| {
3031 !name.starts_with("json:") && !name.starts_with("dependentTasksOutputFiles:")
3032 });
3033 }
3034 let mut always = task.clone();
3035 always.definition.inputs = Some(
3036 declared_inputs(task)
3037 .into_iter()
3038 .filter(|input| input.get("always") == Some(&Value::Bool(true)))
3039 .collect(),
3040 );
3041 let dependencies = dependency_keys(snapshot, workspace, graph, &always, cancelled)?;
3042 let always = self::inputs(
3043 snapshot,
3044 workspace,
3045 &always,
3046 prepared,
3047 &dependencies,
3048 cancelled,
3049 )?;
3050 object.insert(
3051 "always".into(),
3052 json!({"files": always["files"], "values": always["values"], "dependencies": always["dependencies"]}),
3053 );
3054 object.insert("schema".into(), json!("qk-weak-v1"));
3055 object.insert("ignoreReads".into(), json!(ignore_reads));
3056 if !ignored_within.is_empty() {
3061 object.insert("ignoredWithin".into(), json!(ignored_within));
3062 }
3063 key(&weak)
3064}
3065
3066pub(crate) fn dependency_keys(
3071 snapshot: &Snapshot,
3072 workspace: &Workspace,
3073 graph: &TaskGraph,
3074 task: &Task,
3075 cancelled: &AtomicBool,
3076) -> Result<BTreeMap<String, String>> {
3077 dependency_keys_impl(snapshot, workspace, graph, task, cancelled, false)
3078}
3079
3080pub(crate) fn recheck_dependency_keys(
3082 snapshot: &Snapshot,
3083 workspace: &Workspace,
3084 graph: &TaskGraph,
3085 task: &Task,
3086 cancelled: &AtomicBool,
3087) -> Result<BTreeMap<String, String>> {
3088 dependency_keys_impl(snapshot, workspace, graph, task, cancelled, true)
3089}
3090
3091fn dependency_keys_impl(
3092 snapshot: &Snapshot,
3093 workspace: &Workspace,
3094 graph: &TaskGraph,
3095 task: &Task,
3096 cancelled: &AtomicBool,
3097 fresh: bool,
3098) -> Result<BTreeMap<String, String>> {
3099 let resolved = resolve(snapshot, workspace, task, None, cancelled)?;
3100 let selections: Vec<_> = resolved.dependency_output_inputs().collect();
3101 let mut keys = BTreeMap::new();
3102 let mut fresh_outputs = HashMap::new();
3103 for (pattern, transitive) in selections {
3104 let matcher = snapshot.pattern(pattern, false)?;
3105 let mut selected = task.dependencies.clone();
3106 if transitive {
3107 let mut pending = selected.clone();
3108 while let Some(id) = pending.pop_first() {
3109 for dependency in &graph.tasks[&id].dependencies {
3110 if selected.insert(dependency.clone()) {
3111 pending.insert(dependency.clone());
3112 }
3113 }
3114 }
3115 }
3116 for id in selected {
3117 if !Outputs::new(workspace, &graph.tasks[&id])?.declared() {
3119 continue;
3120 }
3121 let outputs = if fresh {
3122 if !fresh_outputs.contains_key(&id) {
3123 fresh_outputs.insert(
3124 id.clone(),
3125 Arc::new(snapshot.fresh_dependency_outputs(workspace, &graph.tasks[&id])?),
3126 );
3127 }
3128 fresh_outputs[&id].clone()
3129 } else {
3130 snapshot.dependency_outputs(workspace, &graph.tasks[&id])?
3131 };
3132 let mut files = BTreeMap::new();
3133 for path in outputs.files.iter().filter(|path| matcher.is_match(path)) {
3134 let known = outputs.values.lock().unwrap().get(path).cloned();
3135 let value = match known {
3136 Some(value) => value,
3137 None => {
3138 let value = snapshot.file_value(&workspace.root, path)?;
3139 if value.get("link").is_none() {
3141 outputs
3142 .values
3143 .lock()
3144 .unwrap()
3145 .insert(path.clone(), value.clone());
3146 }
3147 value
3148 }
3149 };
3150 files.insert(path.clone(), value);
3151 }
3152 keys.insert(format!("{id}:{pattern}"), key(&json!(files))?);
3153 }
3154 }
3155 Ok(keys)
3156}
3157
3158#[derive(Clone, Debug, PartialEq, Eq)]
3161pub struct DependencyGlob {
3162 pub task: String,
3163 pub pattern: String,
3164 pub selects: Selects,
3165}
3166
3167#[derive(Clone, Debug, PartialEq, Eq)]
3168pub enum Selects {
3169 NoDeclaredOutputs,
3171 NotBuilt,
3173 NoneOf(usize),
3175 Some,
3177}
3178
3179pub fn dependency_globs(workspace: &Workspace, graph: &TaskGraph) -> Result<Vec<DependencyGlob>> {
3182 let snapshot = Snapshot::new(workspace, graph, &paths::cache_location(workspace))?;
3183 let cancelled = AtomicBool::new(false);
3184 let mut globs = Vec::new();
3185 for task in graph.tasks.values() {
3186 let Ok(resolved) = resolve(&snapshot, workspace, task, None, &cancelled) else {
3188 continue;
3189 };
3190 for (pattern, transitive) in resolved.dependency_output_inputs() {
3191 let mut selected = task.dependencies.clone();
3192 if transitive {
3193 let mut pending = selected.clone();
3194 while let Some(id) = pending.pop_first() {
3195 for dependency in &graph.tasks[&id].dependencies {
3196 if selected.insert(dependency.clone()) {
3197 pending.insert(dependency.clone());
3198 }
3199 }
3200 }
3201 }
3202 let matcher = snapshot.pattern(pattern, false)?;
3203 let (mut declared, mut files, mut matched, mut unbuilt) = (false, 0, false, false);
3205 for id in &selected {
3206 let dependency = &graph.tasks[id];
3207 if !Outputs::new(workspace, dependency)?.declared() {
3208 continue;
3209 }
3210 declared = true;
3211 let outputs = snapshot.dependency_outputs(workspace, dependency)?;
3212 unbuilt |= outputs.files.is_empty();
3213 files += outputs.files.len();
3214 matched |= outputs.files.iter().any(|path| matcher.is_match(path));
3215 }
3216 globs.push(DependencyGlob {
3217 task: task.id.clone(),
3218 pattern: pattern.to_owned(),
3219 selects: match (declared, matched, unbuilt) {
3220 (false, ..) => Selects::NoDeclaredOutputs,
3221 (_, true, _) => Selects::Some,
3222 (_, _, true) => Selects::NotBuilt,
3223 _ => Selects::NoneOf(files),
3224 },
3225 });
3226 }
3227 }
3228 Ok(globs)
3229}
3230
3231#[cfg(test)]
3232mod tests {
3233
3234 fn whole(resolved: &Resolved) -> BTreeSet<String> {
3236 resolved.whole_files().map(str::to_owned).collect()
3237 }
3238 use super::*;
3239 use qk_taskgraph::Request;
3240
3241 fn fixture(outputs: Option<Value>) -> (tempfile::TempDir, Workspace, TaskGraph) {
3242 let root = tempfile::tempdir().unwrap();
3243 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
3244 let mut post = json!({"command": "true"});
3245 if let Some(outputs) = outputs {
3246 post["outputs"] = outputs;
3247 }
3248 std::fs::write(root.path().join("project.json"), json!({"name":"app", "targets": {
3249 "gen": {"command":"true", "outputs":["{projectRoot}/generated"]},
3250 "post": post,
3251 "build": {"command":"true", "dependsOn":["gen", "post"], "inputs":[{"dependentTasksOutputFiles":"**/*"}]}
3252 }}).to_string()).unwrap();
3253 std::fs::create_dir(root.path().join("generated")).unwrap();
3254 std::fs::write(root.path().join("generated/value"), "one").unwrap();
3255 let workspace = Workspace::load(root.path()).unwrap();
3256 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3257 (root, workspace, graph)
3258 }
3259
3260 #[test]
3264 fn keys_ignore_the_order_object_fields_were_written_in() {
3265 let written = json!({"options": {"cwd": "app", "command": "x"}, "args": ["a", "b"]});
3266 let reordered = json!({"args": ["a", "b"], "options": {"command": "x", "cwd": "app"}});
3267 assert_eq!(key(&written).unwrap(), key(&reordered).unwrap());
3268 let swapped = json!({"options": {"cwd": "app", "command": "x"}, "args": ["b", "a"]});
3269 assert_ne!(key(&written).unwrap(), key(&swapped).unwrap());
3270 }
3271 #[test]
3272 fn json_globs_select_dependency_outputs() {
3273 let root = tempfile::tempdir().unwrap();
3274 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
3275 for (path, content) in [
3276 (
3277 "apps/app/project.json",
3278 json!({"name": "app", "implicitDependencies": ["lib"], "targets": {
3279 "build": {"command": "true", "dependsOn": ["^build"], "inputs": [
3280 {"json": "{workspaceRoot}/libs/lib/out/*.json", "fields": ["version"]}
3281 ]}
3282 }}),
3283 ),
3284 (
3285 "libs/lib/project.json",
3286 json!({"name": "lib", "targets": {
3287 "build": {"command": "true", "outputs": ["{projectRoot}/out"]}
3288 }}),
3289 ),
3290 (
3291 "libs/lib/out/meta.json",
3292 json!({"version": 2, "built": "now"}),
3293 ),
3294 ] {
3295 let path = root.path().join(path);
3296 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
3297 std::fs::write(path, content.to_string()).unwrap();
3298 }
3299 let workspace = Workspace::load(root.path()).unwrap();
3300 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3301 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3302 let resolved = resolve(
3303 &snapshot,
3304 &workspace,
3305 &graph.tasks["app:build"],
3306 None,
3307 &AtomicBool::new(false),
3308 )
3309 .unwrap();
3310 assert_eq!(
3311 resolved.declared.values.get("json:libs/lib/out/meta.json"),
3312 Some(&json!({"version": 2}))
3313 );
3314 assert_eq!(
3315 resolved.declared.producers,
3316 BTreeSet::from(["lib:build".to_owned()])
3317 );
3318 }
3319
3320 #[test]
3324 fn declared_inputs_cover_what_they_would_select_and_not_what_they_leave_out() {
3325 let root = tempfile::tempdir().unwrap();
3326 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
3327 std::fs::write(
3328 root.path().join("project.json"),
3329 json!({"name": "app", "targets": {
3330 "build": {"command": "true", "inputs": [
3331 "{projectRoot}/src/**/*", "!{projectRoot}/src/skip/**/*"
3332 ]},
3333 "gen": {"command": "true", "outputs": ["{projectRoot}/src/gen"]}
3334 }})
3335 .to_string(),
3336 )
3337 .unwrap();
3338 std::fs::write(root.path().join(".gitignore"), "src/secret.ts\n").unwrap();
3339 for path in [
3340 "src/a.ts",
3341 "src/lib/b.ts",
3342 "src/skip/c.ts",
3343 "src/gen/d.ts",
3344 "README.md",
3345 ] {
3346 let path = root.path().join(path);
3347 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
3348 std::fs::write(path, "x").unwrap();
3349 }
3350 let workspace = Workspace::load(root.path()).unwrap();
3351 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3352 let task = &graph.tasks["app:build"];
3353 let prepared = qk_executor::prepare(&workspace, task, &BTreeMap::new()).unwrap();
3354 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3355 let coverage = coverage(
3356 &snapshot,
3357 &workspace,
3358 &graph,
3359 task,
3360 &prepared,
3361 &AtomicBool::new(false),
3362 )
3363 .unwrap();
3364 let set = |part: &str, path: &str| {
3365 let mut set = crate::recorded::PathSet::default();
3366 match part {
3367 "read" => &mut set.reads,
3368 "present" => &mut set.present,
3369 "absent" => &mut set.absent,
3370 _ => &mut set.listings,
3371 }
3372 .insert(path.to_owned());
3373 set
3374 };
3375 for (part, path, covered) in [
3376 ("read", "src/a.ts", true),
3377 ("read", "README.md", false),
3378 ("read", "src/skip/c.ts", false),
3379 ("present", "src/lib", true),
3380 ("absent", "src/new.ts", true),
3381 ("absent", "src/skip/new.ts", false),
3383 ("absent", "src/secret.ts", false),
3384 ("absent", ".env.local", false),
3385 ("absent", "src/gen/e.ts", false),
3387 ("listing", "src/gen", false),
3388 ("listing", "src/lib", true),
3389 ("listing", "src", false),
3391 ("listing", ".", false),
3392 ] {
3393 let generated = |path: &str| path == "src/gen" || path.starts_with("src/gen/");
3394 assert_eq!(
3395 coverage.covers(&snapshot, root.path(), &set(part, path), &generated),
3396 covered,
3397 "{part} {path}"
3398 );
3399 }
3400 }
3401
3402 #[test]
3403 fn execution_fingerprints_refresh_sources_and_preserve_artifact_exclusions() {
3404 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/dist"])));
3405 let definition = workspace
3406 .projects
3407 .get_mut("app")
3408 .unwrap()
3409 .targets
3410 .get_mut("post")
3411 .unwrap();
3412 definition
3413 .extra
3414 .insert("qk:warm".into(), json!({"paths": ["scratch"]}));
3415 std::fs::write(root.path().join(".gitignore"), "ignored/\n").unwrap();
3416 std::fs::create_dir(root.path().join("src")).unwrap();
3417 std::fs::write(root.path().join("src/original"), "one").unwrap();
3418 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3419 let task = &graph.tasks["app:post"];
3420 let prepared = qk_executor::prepare(&workspace, task, &BTreeMap::new()).unwrap();
3421 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3422 let inputs = || {
3423 super::inputs(
3424 &snapshot,
3425 &workspace,
3426 task,
3427 &prepared,
3428 &BTreeMap::new(),
3429 &AtomicBool::new(false),
3430 )
3431 .unwrap()
3432 };
3433 let before = inputs();
3434 for directory in ["generated", "dist", "scratch", "cache", "ignored"] {
3436 std::fs::create_dir_all(root.path().join(directory)).unwrap();
3437 std::fs::write(root.path().join(directory).join("new"), "artifact").unwrap();
3438 }
3439 assert_eq!(before, inputs());
3440 std::fs::write(root.path().join("src/added"), "two").unwrap();
3443 let after = inputs();
3444 assert!(after["files"].get("src/added").is_some());
3445 assert_ne!(key(&before).unwrap(), key(&after).unwrap());
3446 std::fs::remove_file(root.path().join("src/added")).unwrap();
3447 assert_eq!(before, inputs());
3448 std::fs::remove_file(root.path().join("src/original")).unwrap();
3449 assert!(inputs()["files"].get("src/original").is_none());
3450 }
3451
3452 #[test]
3453 fn membership_reuses_settled_candidates_and_detects_directory_changes() {
3454 let mut fixtures = Vec::new();
3455 for git in [false, true] {
3456 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3457 for directory in ["src/empty", "dist", "cache", "ignored"] {
3458 std::fs::create_dir_all(root.path().join(directory)).unwrap();
3459 }
3460 std::fs::write(root.path().join(".gitignore"), "ignored/\n").unwrap();
3461 std::fs::write(root.path().join("src/original"), "one").unwrap();
3462 if git {
3463 assert!(
3464 paths::git(root.path(), &["init", "-q"])
3465 .unwrap()
3466 .status
3467 .success()
3468 );
3469 assert!(
3470 paths::git(root.path(), &["add", "."])
3471 .unwrap()
3472 .status
3473 .success()
3474 );
3475 }
3476 let snapshot =
3477 Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3478 snapshot.current_files(root.path()).unwrap();
3479 fixtures.push((root, snapshot));
3480 }
3481 std::thread::sleep(Duration::from_millis(2100));
3484 for (root, snapshot) in fixtures {
3485 let files = || snapshot.current_files(root.path()).unwrap();
3486 let before = files();
3487 assert!(Arc::ptr_eq(&before, &files()));
3488 for directory in ["generated", "dist", "cache"] {
3489 std::fs::write(root.path().join(directory).join("new"), "artifact").unwrap();
3490 }
3491 assert!(Arc::ptr_eq(&before, &files()));
3492 std::fs::write(root.path().join("src/empty/added"), "two").unwrap();
3493 assert!(files().contains("src/empty/added"));
3494 std::fs::create_dir_all(root.path().join("src/new/nested")).unwrap();
3495 std::fs::write(root.path().join("src/new/nested/added"), "three").unwrap();
3496 assert!(files().contains("src/new/nested/added"));
3497 std::fs::rename(
3498 root.path().join("src/original"),
3499 root.path().join("src/renamed"),
3500 )
3501 .unwrap();
3502 let renamed = files();
3503 assert!(!renamed.contains("src/original"));
3504 assert!(renamed.contains("src/renamed"));
3505 std::fs::remove_file(root.path().join("src/renamed")).unwrap();
3506 assert!(!files().contains("src/renamed"));
3507 std::fs::write(root.path().join(".gitignore"), "ignored/\nsrc/empty/\n").unwrap();
3510 assert!(!files().contains("src/empty/added"));
3511 std::fs::write(root.path().join(".gitignore"), "ignored/\n").unwrap();
3512 assert!(files().contains("src/empty/added"));
3513 }
3514 }
3515
3516 #[test]
3517 fn identical_ignore_rewrites_reuse_candidates_and_changed_rules_refresh() {
3518 for name in [".gitignore", ".nxignore"] {
3519 let (root, workspace, graph) = fixture(None);
3520 let policy = root.path().join(name);
3521 std::fs::write(root.path().join("alpha"), "a").unwrap();
3522 std::fs::write(root.path().join("bravo"), "b").unwrap();
3523 std::fs::write(&policy, "alpha\n").unwrap();
3524 let snapshot =
3525 Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3526 let original = snapshot.current_files(root.path()).unwrap();
3527 assert!(!original.contains("alpha"));
3528 assert!(original.contains("bravo"));
3529 for _ in 0..3 {
3530 std::fs::write(&policy, "alpha\n").unwrap();
3531 assert!(Arc::ptr_eq(
3532 &original,
3533 &snapshot.current_files(root.path()).unwrap()
3534 ));
3535 }
3536 std::fs::write(&policy, "bravo\n").unwrap();
3539 {
3540 let mut cached = snapshot.membership.lock().unwrap();
3541 Arc::make_mut(cached.as_mut().unwrap())
3542 .stamps
3543 .insert(policy.clone(), membership_stamp(&policy).unwrap());
3544 }
3545 let changed = snapshot.current_files(root.path()).unwrap();
3546 assert!(!Arc::ptr_eq(&original, &changed));
3547 assert!(changed.contains("alpha"));
3548 assert!(!changed.contains("bravo"));
3549 std::fs::remove_file(&policy).unwrap();
3550 assert!(
3551 snapshot
3552 .current_files(root.path())
3553 .unwrap()
3554 .contains("bravo")
3555 );
3556 }
3557 }
3558
3559 #[test]
3560 fn ignore_contents_changed_after_inventory_cannot_bless_a_listing() {
3561 let (root, workspace, graph) = fixture(None);
3562 let policy = root.path().join(".nxignore");
3563 std::fs::write(&policy, "alpha\n").unwrap();
3564 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3565 snapshot.current_files(root.path()).unwrap();
3566 let membership = snapshot.membership.lock().unwrap().clone().unwrap();
3567 std::fs::write(&policy, "bravo\n").unwrap();
3568 assert!(!membership.state(None).unwrap().0);
3569 }
3570
3571 #[test]
3572 fn a_directory_missed_by_the_inventory_cannot_enable_reuse() {
3573 let (root, workspace, graph) = fixture(None);
3574 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3575 snapshot.current_files(root.path()).unwrap();
3576 std::fs::create_dir(root.path().join("new-empty")).unwrap();
3577 let mut cached = snapshot.membership.lock().unwrap();
3578 let membership = Arc::make_mut(cached.as_mut().unwrap());
3579 membership
3582 .directory_entries
3583 .get_mut(root.path())
3584 .unwrap()
3585 .insert((OsString::from("new-empty"), 1));
3586 assert!(membership.entries(root.path()).unwrap().is_none());
3587 assert!(!membership.state(None).unwrap().0);
3588 }
3589
3590 #[test]
3591 fn a_policy_missed_by_discovery_cannot_enable_reuse() {
3592 let (root, workspace, graph) = fixture(None);
3593 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3594 snapshot.current_files(root.path()).unwrap();
3595 std::fs::write(root.path().join(".nxignore"), "hidden\n").unwrap();
3596 let mut cached = snapshot.membership.lock().unwrap();
3597 let membership = Arc::make_mut(cached.as_mut().unwrap());
3598 membership
3601 .directory_entries
3602 .get_mut(root.path())
3603 .unwrap()
3604 .insert((OsString::from(".nxignore"), 0));
3605 assert!(membership.entries(root.path()).unwrap().is_none());
3606 assert!(!membership.state(None).unwrap().0);
3607 }
3608
3609 #[test]
3610 fn output_publication_reuses_candidates_but_source_names_still_refresh() {
3611 for partial in [false, true] {
3612 let outputs = if partial {
3613 json!(["{projectRoot}/dist", "!{projectRoot}/dist/keep"])
3614 } else {
3615 json!(["{projectRoot}/dist"])
3616 };
3617 let (root, workspace, graph) = fixture(Some(outputs));
3618 std::fs::create_dir_all(root.path().join("src")).unwrap();
3619 std::fs::write(root.path().join("src/original"), "one").unwrap();
3620 if partial {
3621 std::fs::create_dir_all(root.path().join("dist/keep")).unwrap();
3622 }
3623 let snapshot =
3624 Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3625 snapshot.current_files(root.path()).unwrap();
3626 std::thread::sleep(Duration::from_millis(2100));
3629 let before = snapshot.current_files(root.path()).unwrap();
3630 std::fs::create_dir_all(root.path().join("dist")).unwrap();
3631 std::fs::write(root.path().join("dist/restored"), "artifact").unwrap();
3632 let restored = snapshot.current_files(root.path()).unwrap();
3633 assert!(Arc::ptr_eq(&before, &restored));
3634 std::fs::remove_file(root.path().join("dist/restored")).unwrap();
3635 assert!(Arc::ptr_eq(
3636 &before,
3637 &snapshot.current_files(root.path()).unwrap()
3638 ));
3639 let source = if partial { "dist/keep/new" } else { "src/new" };
3640 std::fs::write(root.path().join(source), "source").unwrap();
3641 let after = snapshot.current_files(root.path()).unwrap();
3642 assert!(after.contains(source));
3643 assert!(!Arc::ptr_eq(&before, &after));
3644 std::fs::remove_file(root.path().join(source)).unwrap();
3645 assert!(
3646 !snapshot
3647 .current_files(root.path())
3648 .unwrap()
3649 .contains(source)
3650 );
3651 }
3652 }
3653
3654 #[test]
3655 fn concurrent_fingerprints_refresh_without_replacing_newer_membership() {
3656 let (root, workspace, graph) = fixture(None);
3657 std::fs::create_dir_all(root.path().join("src")).unwrap();
3658 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3659 let original = snapshot.current_files(root.path()).unwrap();
3660 let barrier = std::sync::Barrier::new(9);
3661 std::thread::scope(|scope| {
3662 let workers: Vec<_> = (0..8)
3663 .map(|_| {
3664 scope.spawn(|| {
3665 let mut rounds = Vec::new();
3666 for _ in 0..6 {
3667 barrier.wait();
3668 rounds.push(snapshot.current_files(root.path()));
3669 barrier.wait();
3670 }
3671 rounds
3672 })
3673 })
3674 .collect();
3675 for round in 0..6 {
3676 std::fs::write(root.path().join(format!("src/new{round}")), "source").unwrap();
3677 barrier.wait();
3678 barrier.wait();
3679 }
3680 for worker in workers {
3683 for (round, files) in worker.join().unwrap().into_iter().enumerate() {
3684 let files = files.unwrap();
3685 for added in 0..=round {
3686 assert!(files.contains(&format!("src/new{added}")));
3687 }
3688 }
3689 }
3690 });
3691 assert!(!original.contains("src/new0"));
3692 assert!(
3693 snapshot
3694 .current_files(root.path())
3695 .unwrap()
3696 .contains("src/new5")
3697 );
3698 }
3699
3700 #[test]
3701 fn retained_candidates_keep_their_dependency_output_ignore_policy() {
3702 let (root, mut workspace, _) = fixture(None);
3703 workspace
3704 .projects
3705 .get_mut("app")
3706 .unwrap()
3707 .targets
3708 .get_mut("build")
3709 .unwrap()
3710 .inputs = Some(vec![json!("{workspaceRoot}/**/*.txt")]);
3711 std::fs::create_dir(root.path().join("src")).unwrap();
3712 for name in ["alpha", "beta", "gamma"] {
3713 for directory in ["src", "generated"] {
3714 std::fs::write(root.path().join(format!("{directory}/{name}.txt")), name).unwrap();
3715 }
3716 }
3717 let policy = |keep: &str| {
3718 let ignored: String = ["alpha", "beta", "gamma"]
3719 .into_iter()
3720 .filter(|name| *name != keep)
3721 .flat_map(|name| {
3722 [
3723 format!("src/{name}.txt\n"),
3724 format!("generated/{name}.txt\n"),
3725 ]
3726 })
3727 .collect();
3728 std::fs::write(root.path().join(".nxignore"), ignored).unwrap();
3729 };
3730 policy("alpha");
3731 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3732 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
3733 let first = snapshot
3734 .current_files_for(root.path(), None, "test")
3735 .unwrap();
3736 policy("beta");
3737 let second = snapshot
3738 .current_files_for(root.path(), None, "test")
3739 .unwrap();
3740 policy("gamma");
3741 let third = snapshot
3742 .current_files_for(root.path(), None, "test")
3743 .unwrap();
3744 let cancelled = AtomicBool::new(false);
3747 let expected = |name: &str| {
3748 BTreeSet::from([
3749 ".nxignore".into(),
3750 format!("src/{name}.txt"),
3751 format!("generated/{name}.txt"),
3752 ])
3753 };
3754 for (candidates, name) in [(first, "alpha"), (second, "beta"), (third, "gamma")] {
3755 let resolved = resolve_candidates(
3756 &snapshot,
3757 &workspace,
3758 &graph.tasks["app:build"],
3759 None,
3760 &cancelled,
3761 &candidates.files,
3762 &candidates.source_ignore,
3763 )
3764 .unwrap();
3765 assert_eq!(whole(&resolved), expected(name));
3766 }
3767 let inspected = resolve(
3769 &snapshot,
3770 &workspace,
3771 &graph.tasks["app:build"],
3772 None,
3773 &cancelled,
3774 )
3775 .unwrap();
3776 assert_eq!(whole(&inspected), expected("alpha"));
3777 }
3778
3779 #[test]
3780 fn scoped_membership_defers_unrelated_changes_until_a_matching_task_asks() {
3781 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/dist"])));
3782 let targets = &mut workspace.projects.get_mut("app").unwrap().targets;
3783 targets.get_mut("post").unwrap().inputs =
3784 Some(vec![json!("{workspaceRoot}/left/**/*.txt")]);
3785 targets.get_mut("gen").unwrap().inputs =
3786 Some(vec![json!("{workspaceRoot}/right/**/*.txt")]);
3787 targets.get_mut("build").unwrap().inputs = Some(vec![json!("{workspaceRoot}/**/*.txt")]);
3788 for directory in ["left/empty", "right/empty"] {
3789 std::fs::create_dir_all(root.path().join(directory)).unwrap();
3790 }
3791 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3792 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3793 snapshot.current_files(&workspace.root).unwrap();
3794 std::thread::sleep(Duration::from_millis(2100));
3795 let left = snapshot
3796 .membership_prefixes(&workspace, &graph.tasks["app:post"])
3797 .unwrap();
3798 let right = snapshot
3799 .membership_prefixes(&workspace, &graph.tasks["app:gen"])
3800 .unwrap();
3801 let all = snapshot
3802 .membership_prefixes(&workspace, &graph.tasks["app:build"])
3803 .unwrap();
3804 let before = snapshot
3805 .current_files_for(&workspace.root, Some(&left), "test")
3806 .unwrap();
3807 std::fs::write(root.path().join("right/empty/new.txt"), "right").unwrap();
3808 let unrelated = snapshot
3809 .current_files_for(&workspace.root, Some(&left), "test")
3810 .unwrap();
3811 assert!(Arc::ptr_eq(&before.files, &unrelated.files));
3812 assert!(!unrelated.files.contains("right/empty/new.txt"));
3813 assert!(
3814 snapshot
3815 .current_files_for(&workspace.root, Some(&right), "test")
3816 .unwrap()
3817 .files
3818 .contains("right/empty/new.txt")
3819 );
3820 std::fs::create_dir_all(root.path().join("left/empty/nested")).unwrap();
3821 std::fs::write(root.path().join("left/empty/nested/new.txt"), "left").unwrap();
3822 assert!(
3823 snapshot
3824 .current_files_for(&workspace.root, Some(&left), "test")
3825 .unwrap()
3826 .files
3827 .contains("left/empty/nested/new.txt")
3828 );
3829 std::fs::write(root.path().join("right/empty/another.txt"), "right").unwrap();
3830 assert!(
3831 snapshot
3832 .current_files_for(&workspace.root, Some(&all), "test")
3833 .unwrap()
3834 .files
3835 .contains("right/empty/another.txt")
3836 );
3837 }
3838
3839 #[test]
3840 fn membership_watches_the_directories_json_globs_select_from() {
3841 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/dist"])));
3842 let targets = &mut workspace.projects.get_mut("app").unwrap().targets;
3843 targets.get_mut("build").unwrap().inputs = Some(vec![json!(
3844 {"json": "{workspaceRoot}/config/**/*.json", "fields": ["version"]}
3845 )]);
3846 std::fs::create_dir_all(root.path().join("config/empty")).unwrap();
3847 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3848 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3849 snapshot.current_files(&workspace.root).unwrap();
3850 std::thread::sleep(Duration::from_millis(2100));
3851 let prefixes = snapshot
3852 .membership_prefixes(&workspace, &graph.tasks["app:build"])
3853 .unwrap();
3854 snapshot
3855 .current_files_for(&workspace.root, Some(&prefixes), "test")
3856 .unwrap();
3857 std::fs::write(
3858 root.path().join("config/empty/new.json"),
3859 r#"{"version": 1}"#,
3860 )
3861 .unwrap();
3862 assert!(
3863 snapshot
3864 .current_files_for(&workspace.root, Some(&prefixes), "test")
3865 .unwrap()
3866 .files
3867 .contains("config/empty/new.json")
3868 );
3869 }
3870
3871 #[test]
3872 fn membership_watches_git_ignored_tracked_parents_and_index_policy() {
3873 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3874 std::fs::create_dir_all(root.path().join("tracked/nested")).unwrap();
3875 std::fs::write(root.path().join("tracked/.gitignore"), "nested/\n").unwrap();
3876 std::fs::write(root.path().join("tracked/nested/source"), "one").unwrap();
3877 assert!(
3878 paths::git(root.path(), &["init", "-q"])
3879 .unwrap()
3880 .status
3881 .success()
3882 );
3883 assert!(
3884 paths::git(root.path(), &["add", "-f", "tracked/nested/source"])
3885 .unwrap()
3886 .status
3887 .success()
3888 );
3889 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3890 let files = || snapshot.current_files(root.path()).unwrap();
3891 assert!(files().contains("tracked/nested/source"));
3892 std::fs::remove_file(root.path().join("tracked/nested/source")).unwrap();
3893 assert!(!files().contains("tracked/nested/source"));
3894 std::fs::write(root.path().join("tracked/nested/another"), "two").unwrap();
3895 assert!(!files().contains("tracked/nested/another"));
3896 assert!(
3897 paths::git(root.path(), &["add", "-f", "tracked/nested/another"])
3898 .unwrap()
3899 .status
3900 .success()
3901 );
3902 assert!(files().contains("tracked/nested/another"));
3903 std::fs::write(root.path().join("untracked"), "three").unwrap();
3904 assert!(files().contains("untracked"));
3905 std::fs::write(root.path().join(".git/info/exclude"), "untracked\n").unwrap();
3906 assert!(!files().contains("untracked"));
3907 }
3908
3909 #[test]
3910 fn a_new_git_config_include_invalidates_settled_listing_policy() {
3911 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3912 let policy = tempfile::tempdir().unwrap();
3913 let included = policy.path().join("included.config");
3914 let excludes = policy.path().join("excludes");
3915 std::fs::write(&excludes, "hidden\n").unwrap();
3916 std::fs::write(root.path().join("hidden"), "source").unwrap();
3917 assert!(
3918 paths::git(root.path(), &["init", "-q"])
3919 .unwrap()
3920 .status
3921 .success()
3922 );
3923 assert!(
3924 paths::git(
3925 root.path(),
3926 &["config", "include.path", included.to_str().unwrap()]
3927 )
3928 .unwrap()
3929 .status
3930 .success()
3931 );
3932 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3933 snapshot.current_files(&workspace.root).unwrap();
3934 std::thread::sleep(Duration::from_millis(2100));
3935 let before = snapshot.current_files(&workspace.root).unwrap();
3936 assert!(before.contains("hidden"));
3937 assert!(Arc::ptr_eq(
3938 &before,
3939 &snapshot.current_files(&workspace.root).unwrap()
3940 ));
3941 assert!(
3942 paths::git(
3943 root.path(),
3944 &[
3945 "config",
3946 "--file",
3947 included.to_str().unwrap(),
3948 "core.excludesfile",
3949 excludes.to_str().unwrap(),
3950 ]
3951 )
3952 .unwrap()
3953 .status
3954 .success()
3955 );
3956 assert!(
3957 !snapshot
3958 .current_files(&workspace.root)
3959 .unwrap()
3960 .contains("hidden")
3961 );
3962 }
3963
3964 #[cfg(unix)]
3965 #[test]
3966 fn git_policy_paths_preserve_quoted_origins_and_multiline_values() {
3967 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3968 let policy = tempfile::tempdir().unwrap();
3969 let included = policy.path().join("included\\é\t\n.config");
3971 let excludes = policy.path().join("excludes\\é\t\n");
3972 let replacement = policy.path().join("replacement\n");
3973 std::fs::write(&excludes, "hidden\n").unwrap();
3974 std::fs::write(&replacement, "visible\n").unwrap();
3975 for file in ["hidden", "visible"] {
3976 std::fs::write(root.path().join(file), "source").unwrap();
3977 }
3978 let git = |args: &[&str]| {
3979 let output = paths::git(root.path(), args).unwrap();
3980 assert!(output.status.success(), "{output:?}");
3981 output
3982 };
3983 git(&["init", "-q"]);
3984 git(&["config", "include.path", included.to_str().unwrap()]);
3985 git(&[
3986 "config",
3987 "--file",
3988 included.to_str().unwrap(),
3989 "core.excludesfile",
3990 excludes.to_str().unwrap(),
3991 ]);
3992 let text = git(&["config", "--show-origin", "--list"]);
3993 let text = String::from_utf8_lossy(&text.stdout);
3994 let quoted_origin = text
3995 .lines()
3996 .find_map(|line| {
3997 line.strip_prefix("file:\"")
3998 .and_then(|_| line.split_once('\t'))
3999 })
4000 .unwrap()
4001 .0
4002 .strip_prefix("file:")
4003 .unwrap();
4004 let policies = git_membership_policies(root.path());
4005 assert!(!policies.contains(&root.path().join(quoted_origin)));
4006 assert!(policies.contains(&included));
4007 assert!(policies.contains(&excludes));
4008 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
4009 let initial = snapshot.current_files(&workspace.root).unwrap();
4010 assert!(!initial.contains("hidden"));
4011 assert!(initial.contains("visible"));
4012 std::thread::sleep(Duration::from_millis(2100));
4013 let settled = snapshot.current_files(&workspace.root).unwrap();
4014 assert!(Arc::ptr_eq(
4015 &settled,
4016 &snapshot.current_files(&workspace.root).unwrap()
4017 ));
4018 git(&[
4019 "config",
4020 "--file",
4021 included.to_str().unwrap(),
4022 "core.excludesfile",
4023 replacement.to_str().unwrap(),
4024 ]);
4025 let changed = snapshot.current_files(&workspace.root).unwrap();
4026 assert!(changed.contains("hidden"));
4027 assert!(!changed.contains("visible"));
4028 assert!(git_membership_policies(root.path()).contains(&replacement));
4029 }
4030
4031 #[cfg(unix)]
4032 #[test]
4033 fn symlinked_ignore_policy_never_reuses_stale_membership() {
4034 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
4035 let policy = tempfile::NamedTempFile::new().unwrap();
4036 std::fs::write(policy.path(), "hidden\n").unwrap();
4037 std::os::unix::fs::symlink(policy.path(), root.path().join(".gitignore")).unwrap();
4038 std::fs::write(root.path().join("hidden"), "source").unwrap();
4039 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
4040 assert!(
4041 !snapshot
4042 .current_files(root.path())
4043 .unwrap()
4044 .contains("hidden")
4045 );
4046 assert!(
4047 !snapshot
4048 .membership
4049 .lock()
4050 .unwrap()
4051 .as_ref()
4052 .unwrap()
4053 .settled
4054 );
4055 std::thread::sleep(Duration::from_millis(2100));
4056 let before = snapshot.current_files(root.path()).unwrap();
4057 assert!(Arc::ptr_eq(
4058 &before,
4059 &snapshot.current_files(root.path()).unwrap()
4060 ));
4061 std::fs::write(policy.path(), "").unwrap();
4062 assert!(
4063 snapshot
4064 .current_files(root.path())
4065 .unwrap()
4066 .contains("hidden")
4067 );
4068 }
4069
4070 #[cfg(unix)]
4074 #[test]
4075 fn a_linked_directory_input_keys_ignored_files_beneath_it() {
4076 let root = tempfile::tempdir().unwrap();
4077 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
4078 std::fs::write(
4079 root.path().join("project.json"),
4080 json!({"name": "app", "targets": {
4081 "read": {"command": "true", "inputs": ["{projectRoot}/linked"]},
4082 "whole": {"command": "true", "inputs": ["{projectRoot}/everything"]}
4083 }})
4084 .to_string(),
4085 )
4086 .unwrap();
4087 std::fs::write(root.path().join(".gitignore"), ".env\n").unwrap();
4088 std::fs::create_dir(root.path().join("shared")).unwrap();
4089 std::fs::write(root.path().join("shared/a.ts"), "a").unwrap();
4090 std::fs::write(root.path().join("shared/.env"), "X=1\n").unwrap();
4091 std::os::unix::fs::symlink("shared", root.path().join("linked")).unwrap();
4092 std::os::unix::fs::symlink(".", root.path().join("everything")).unwrap();
4093 assert!(
4094 paths::git(root.path(), &["init", "--quiet"])
4095 .unwrap()
4096 .status
4097 .success()
4098 );
4099 let workspace = Workspace::load(root.path()).unwrap();
4100 let requests = ["app:read", "app:whole"].map(|request| Request::parse(request).unwrap());
4101 let graph = TaskGraph::build(&workspace, &requests).unwrap();
4102 let key = |id: &str| {
4103 let task = &graph.tasks[id];
4104 let prepared = qk_executor::prepare(&workspace, task, &BTreeMap::new()).unwrap();
4105 let snapshot =
4106 Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
4107 inputs(
4108 &snapshot,
4109 &workspace,
4110 task,
4111 &prepared,
4112 &BTreeMap::new(),
4113 &AtomicBool::new(false),
4114 )
4115 };
4116 let before = key("app:read").unwrap();
4117 std::fs::write(root.path().join("shared/.env"), "X=2\n").unwrap();
4118 assert_ne!(before, key("app:read").unwrap());
4119 let error = format!("{:#}", key("app:whole").unwrap_err());
4120 assert!(error.contains("resolves to the workspace root"), "{error}");
4121 }
4122
4123 #[cfg(unix)]
4124 #[test]
4125 fn recent_change_time_prevents_membership_reuse_despite_old_mtime() {
4126 use std::os::unix::fs::MetadataExt;
4127 let root = tempfile::tempdir().unwrap();
4128 let old = SystemTime::UNIX_EPOCH + Duration::from_secs(1);
4129 File::open(root.path())
4130 .unwrap()
4131 .set_times(std::fs::FileTimes::new().set_modified(old))
4132 .unwrap();
4133 let metadata = std::fs::metadata(root.path()).unwrap();
4134 assert!(metadata.ctime() > 1);
4135 let stamp = Stamp::new(&metadata);
4136 assert!(stamp.settled());
4137 assert!(!stamp.membership_settled());
4138 }
4139
4140 #[test]
4141 fn inspection_keeps_deleted_candidates() {
4142 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
4143 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache"))
4144 .unwrap()
4145 .with_candidates(&["src/deleted".into()]);
4146 let resolved = resolve(
4147 &snapshot,
4148 &workspace,
4149 &graph.tasks["app:post"],
4150 None,
4151 &AtomicBool::new(false),
4152 )
4153 .unwrap();
4154 assert!(resolved.declared.files.contains("src/deleted"));
4155 }
4156
4157 #[test]
4158 fn undeclared_writes_invalidate_dependency_values() {
4159 for outputs in [None, Some(json!(["{projectRoot}/generated"]))] {
4160 let (root, workspace, graph) = fixture(outputs);
4161 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
4162 let keys = || {
4163 dependency_keys(
4164 &snapshot,
4165 &workspace,
4166 &graph,
4167 &graph.tasks["app:build"],
4168 &AtomicBool::new(false),
4169 )
4170 .unwrap()
4171 };
4172 let before = keys();
4173 std::fs::write(root.path().join("generated/value"), "different").unwrap();
4174 snapshot.outputs_written(&workspace, &graph.tasks["app:post"]);
4175 assert_ne!(before, keys());
4176 }
4177 }
4178
4179 #[cfg(unix)]
4180 #[test]
4181 fn dependency_symlinks_recheck_targets_outside_output_roots() {
4182 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/generated"])));
4183 workspace
4184 .projects
4185 .get_mut("app")
4186 .unwrap()
4187 .targets
4188 .get_mut("gen")
4189 .unwrap()
4190 .outputs = Some(vec!["{projectRoot}/links".into()]);
4191 std::fs::create_dir(root.path().join("links")).unwrap();
4192 std::os::unix::fs::symlink("../generated/value", root.path().join("links/value")).unwrap();
4193 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
4194 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
4195 let keys = || {
4196 dependency_keys(
4197 &snapshot,
4198 &workspace,
4199 &graph,
4200 &graph.tasks["app:build"],
4201 &AtomicBool::new(false),
4202 )
4203 .unwrap()
4204 };
4205 let before = keys();
4206 std::fs::write(root.path().join("generated/value"), "different").unwrap();
4207 snapshot.outputs_written(&workspace, &graph.tasks["app:post"]);
4208 assert_ne!(before["app:gen:**/*"], keys()["app:gen:**/*"]);
4209 }
4210 #[test]
4213 fn digests_of_ignored_files_read_by_include_ignored_are_saved() {
4214 let root = tempfile::tempdir().unwrap();
4215 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
4216 std::fs::write(root.path().join(".gitignore"), "generated/\n").unwrap();
4217 std::fs::write(
4218 root.path().join("project.json"),
4219 json!({"name": "app", "targets": {"build": {"command": "true", "inputs": [
4220 {"fileset": "{projectRoot}/generated/**/*", "includeIgnored": true}
4221 ]}}})
4222 .to_string(),
4223 )
4224 .unwrap();
4225 std::fs::create_dir(root.path().join("generated")).unwrap();
4226 std::fs::write(root.path().join("generated/schema.json"), "{}").unwrap();
4227 let old = std::time::SystemTime::now() - std::time::Duration::from_secs(60);
4229 std::fs::File::options()
4230 .write(true)
4231 .open(root.path().join("generated/schema.json"))
4232 .unwrap()
4233 .set_times(std::fs::FileTimes::new().set_modified(old))
4234 .unwrap();
4235 let git = |args: &[&str]| assert!(paths::git(root.path(), args).unwrap().status.success());
4236 git(&["init", "--quiet"]);
4237 let workspace = Workspace::load(root.path()).unwrap();
4238 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
4239 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
4240 assert!(!snapshot.files.contains("generated/schema.json"));
4241 let prepared =
4242 qk_executor::prepare(&workspace, &graph.tasks["app:build"], &BTreeMap::new()).unwrap();
4243 inputs(
4244 &snapshot,
4245 &workspace,
4246 &graph.tasks["app:build"],
4247 &prepared,
4248 &BTreeMap::new(),
4249 &AtomicBool::new(false),
4250 )
4251 .unwrap();
4252 snapshot.save_digests(root.path());
4253 assert!(load_digests(root.path()).contains_key("generated/schema.json"));
4254 }
4255}