1use std::collections::{BTreeMap, BTreeSet};
2use std::ffi::OsString;
3use std::fs::File;
4use std::io::{Read, Write};
5use std::path::{Path, PathBuf};
6use std::sync::atomic::AtomicBool;
7use std::sync::{Arc, Mutex};
8use std::time::{Duration, Instant, SystemTime};
9
10use anyhow::{Context, Result, bail};
11use foldhash::{HashMap, HashMapExt, HashSet};
12use qk_config::Workspace;
13use qk_executor::{Capture, Display, Outcome, PreparedTask, execute_captured, read_capture};
14use qk_graph::ProjectGraph;
15use qk_lockfile::Lockfile;
16use qk_taskgraph::{Task, TaskGraph};
17use serde_json::{Value, json};
18
19use crate::ignored::Group;
20use crate::inputs::{
21 Declared, Inferred, JsonFields, Resolved, TsConfig, manifest_for_dependents,
22 workspace_without_resolution,
23};
24use crate::paths::{self, Outputs, Pattern};
25
26pub fn digest_file(path: &Path) -> Result<String> {
27 let mut hasher = blake3::Hasher::new();
28 let mut reader = File::open(path)?;
29 let mut buffer = [0; 64 * 1024];
30 loop {
31 let count = reader.read(&mut buffer)?;
32 if count == 0 {
33 break;
34 }
35 hasher.update(&buffer[..count]);
36 }
37 Ok(hasher.finalize().to_hex().to_string())
38}
39
40#[derive(Clone, PartialEq, serde::Serialize, serde::Deserialize)]
43struct Stamp(Vec<i64>);
44
45impl Stamp {
46 fn new(metadata: &std::fs::Metadata) -> Self {
47 let modified = metadata
48 .modified()
49 .ok()
50 .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok())
51 .map_or(-1, |duration| duration.as_nanos() as i64);
52 #[allow(unused_mut)]
53 let mut values = vec![metadata.len() as i64, modified];
54 #[cfg(unix)]
55 {
56 use std::os::unix::fs::MetadataExt;
57 values.extend([
58 metadata.ctime(),
59 metadata.ctime_nsec(),
60 metadata.ino() as i64,
61 i64::from(metadata.mode()),
62 ]);
63 }
64 Self(values)
65 }
66
67 fn membership_settled(&self) -> bool {
70 #[cfg(unix)]
71 let width = 6;
72 #[cfg(not(unix))]
73 let width = 2;
74 self.0.chunks_exact(width).all(|values| {
75 let modified = SystemTime::UNIX_EPOCH + Duration::from_nanos(values[1].max(0) as u64);
76 let settled = |time| {
77 SystemTime::now()
78 .duration_since(time)
79 .is_ok_and(|age| age > Duration::from_secs(2))
80 };
81 if values[1] < 0 || !settled(modified) {
82 return false;
83 }
84 #[cfg(unix)]
85 {
86 let changed = SystemTime::UNIX_EPOCH
87 + Duration::new(values[2].max(0) as u64, values[3].max(0) as u32);
88 values[2] >= 0 && settled(changed)
89 }
90 #[cfg(not(unix))]
91 true
92 })
93 }
94
95 fn settled(&self) -> bool {
98 let modified = SystemTime::UNIX_EPOCH + Duration::from_nanos(self.0[1].max(0) as u64);
99 self.0[1] >= 0
100 && SystemTime::now()
101 .duration_since(modified)
102 .is_ok_and(|age| age > Duration::from_secs(2))
103 }
104}
105
106fn digests_path(root: &Path) -> PathBuf {
110 paths::worktree_state(root).join("digests.json")
111}
112
113#[derive(PartialEq, Eq, Hash)]
114struct RuntimeKey {
115 command: String,
116 environment: BTreeMap<OsString, OsString>,
117}
118
119pub struct Snapshot {
123 pub(crate) files: BTreeSet<String>,
124 extra_candidates: BTreeSet<String>,
125 membership: Mutex<Option<Arc<Membership>>>,
126 membership_prefixes: Mutex<HashMap<String, Arc<Vec<PathBuf>>>>,
127 source_exclusions: Vec<Outputs>,
128 cache_relative: Option<PathBuf>,
129 projects: ProjectGraph,
130 roots: BTreeMap<String, String>,
132 generated_tasks: Vec<(Task, Outputs)>,
133 task_dependencies: BTreeMap<String, BTreeSet<String>>,
134 servers: BTreeMap<String, Vec<(String, Vec<Value>)>>,
136 source_ignore: Arc<SourceIgnore>,
137 canonical_root: PathBuf,
138 workspace_prefix: Option<PathBuf>,
139 patterns: Mutex<HashMap<(String, bool, bool), Arc<Pattern>>>,
140 digests: Mutex<HashMap<String, (Stamp, String)>>,
141 ignored_files: Mutex<BTreeSet<String>>,
144 digests_changed: std::sync::atomic::AtomicBool,
146 directories: Mutex<HashSet<PathBuf>>,
148 runtime: Mutex<HashMap<RuntimeKey, Arc<Mutex<Option<Value>>>>>,
150 lockfile: Mutex<Option<Arc<Installed>>>,
152 dependency_outputs: Mutex<HashMap<String, Arc<DependencyOutputs>>>,
155 outputs_written: std::sync::atomic::AtomicU64,
157}
158
159#[derive(Clone)]
162struct Membership {
163 files: Arc<BTreeSet<String>>,
164 source_ignore: Arc<SourceIgnore>,
165 stamps: BTreeMap<PathBuf, Option<Stamp>>,
166 directories: BTreeSet<PathBuf>,
167 directory_entries: BTreeMap<PathBuf, BTreeSet<(OsString, u8)>>,
168 root: PathBuf,
169 outputs: Vec<Outputs>,
170 output_anchors: BTreeMap<PathBuf, Vec<usize>>,
171 excluded_roots: BTreeSet<PathBuf>,
172 cache: Option<PathBuf>,
173 policies: BTreeSet<PathBuf>,
174 git_policies: BTreeSet<PathBuf>,
175 ignore_contents: BTreeMap<PathBuf, String>,
176 settled: bool,
177 listed: Option<Instant>,
179}
180
181#[derive(Clone)]
184struct Candidates {
185 files: Arc<BTreeSet<String>>,
186 source_ignore: Arc<SourceIgnore>,
187}
188
189impl Membership {
190 fn new(
192 root: &Path,
193 files: &BTreeSet<String>,
194 source_ignore: &Arc<SourceIgnore>,
195 outputs: &[Outputs],
196 cache: Option<&Path>,
197 git_policies: Option<&BTreeSet<PathBuf>>,
198 ) -> Result<Self> {
199 let excluded: BTreeSet<_> = outputs
200 .iter()
201 .flat_map(Outputs::complete_roots)
202 .map(PathBuf::from)
203 .collect();
204 let excluded_roots = excluded.clone();
205 let walk_root = root.to_owned();
206 let walk_cache = cache.map(PathBuf::from);
207 let mut directories = BTreeSet::new();
208 let mut stamps = BTreeMap::new();
209 for entry in ignore::WalkBuilder::new(root)
210 .hidden(false)
211 .parents(false)
212 .git_global(false)
213 .require_git(false)
214 .ignore(false)
215 .git_ignore(false)
216 .git_exclude(false)
217 .follow_links(false)
218 .filter_entry(move |entry| {
219 let relative = entry
220 .path()
221 .strip_prefix(&walk_root)
222 .unwrap_or(entry.path());
223 !matches!(
224 entry.file_name().to_str(),
225 Some(".git" | ".qk" | "node_modules")
226 ) && walk_cache
227 .as_ref()
228 .is_none_or(|cache| !relative.starts_with(cache))
229 && !relative.ancestors().any(|path| excluded.contains(path))
230 })
231 .build()
232 {
233 let entry = entry?;
234 if entry.file_type().is_some_and(|kind| kind.is_dir()) {
235 let directory = entry.path().to_owned();
236 stamps.insert(directory.clone(), membership_stamp(&directory)?);
237 directories.insert(directory);
238 }
239 }
240 for path in files {
243 for parent in Path::new(path).ancestors().skip(1) {
244 let directory = root.join(parent);
245 if directories.insert(directory.clone()) {
246 stamps.insert(directory.clone(), membership_stamp(&directory)?);
247 }
248 }
249 }
250 let mut policies = BTreeSet::new();
251 for directory in &directories {
252 for name in [".gitignore", ".nxignore"] {
253 let path = directory.join(name);
254 if membership_stamp(&path)?.is_some() {
257 policies.insert(path);
258 }
259 }
260 }
261 for directory in root.ancestors() {
262 policies.insert(directory.join(".gitignore"));
263 policies.insert(directory.join(".git"));
265 }
266 let git_policies = git_policies
267 .cloned()
268 .unwrap_or_else(|| git_membership_policies(root));
269 policies.extend(git_policies.iter().cloned());
270 for policy in &policies {
271 stamps.insert(policy.clone(), membership_stamp(policy)?);
272 }
273 let mut ignore_contents = BTreeMap::new();
276 for policy in &policies {
277 if matches!(
278 policy.file_name().and_then(|name| name.to_str()),
279 Some(".gitignore" | ".nxignore")
280 ) && let Some(content) = ignore_content(policy, &stamps[policy])?
281 {
282 ignore_contents.insert(policy.clone(), content);
283 }
284 }
285 let mut output_anchors: BTreeMap<PathBuf, Vec<usize>> = BTreeMap::new();
286 for (index, outputs) in outputs.iter().enumerate() {
287 for anchor in outputs.anchors() {
288 output_anchors
289 .entry(PathBuf::from(anchor))
290 .or_default()
291 .push(index);
292 }
293 }
294 let mut membership = Self {
295 files: Arc::new(BTreeSet::new()),
296 source_ignore: source_ignore.clone(),
297 stamps,
298 directories,
299 directory_entries: BTreeMap::new(),
300 root: root.to_owned(),
301 outputs: outputs.to_vec(),
302 output_anchors,
303 excluded_roots,
304 cache: cache.map(PathBuf::from),
305 policies,
306 git_policies,
307 ignore_contents,
308 settled: false,
309 listed: None,
310 };
311 for directory in &membership.directories {
312 if let Some(entries) = membership.entries(directory)? {
313 membership
314 .directory_entries
315 .insert(directory.clone(), entries);
316 }
317 }
318 Ok(membership)
319 }
320
321 fn entries(&self, directory: &Path) -> Result<Option<BTreeSet<(OsString, u8)>>> {
325 let entries = match std::fs::read_dir(directory) {
326 Ok(entries) => entries,
327 Err(error)
328 if matches!(
329 error.kind(),
330 std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
331 ) =>
332 {
333 return Ok(None);
334 }
335 Err(error) => return Err(error.into()),
336 };
337 let relative_directory = directory.strip_prefix(&self.root).unwrap_or(directory);
338 let mut outputs = BTreeSet::new();
339 for ancestor in relative_directory.ancestors() {
340 if let Some(indices) = self.output_anchors.get(ancestor) {
341 outputs.extend(indices.iter().copied());
342 }
343 }
344 for (_, indices) in self
345 .output_anchors
346 .range(relative_directory.to_owned()..)
347 .take_while(|(anchor, _)| anchor.starts_with(relative_directory))
348 {
349 outputs.extend(indices.iter().copied());
350 }
351 let mut names = BTreeSet::new();
352 for entry in entries {
353 let entry = entry?;
354 let name = entry.file_name();
355 let path = entry.path();
356 let relative = path.strip_prefix(&self.root).unwrap_or(&path);
357 if matches!(name.to_str(), Some(".git" | ".qk" | "node_modules"))
358 || self
359 .cache
360 .as_ref()
361 .is_some_and(|cache| relative.starts_with(cache))
362 || relative
363 .ancestors()
364 .any(|root| self.excluded_roots.contains(root))
365 {
366 continue;
367 }
368 if matches!(name.to_str(), Some(".gitignore" | ".nxignore"))
371 && !self.policies.contains(&path)
372 {
373 return Ok(None);
374 }
375 let kind = entry.file_type()?;
376 if kind.is_dir() && !self.directories.contains(&path) {
380 return Ok(None);
381 }
382 if !kind.is_dir()
383 && !matches!(name.to_str(), Some(".gitignore" | ".nxignore"))
384 && relative.to_str().is_some_and(|path| {
385 outputs
386 .iter()
387 .any(|index| self.outputs[*index].matches(path))
388 })
389 {
390 continue;
391 }
392 names.insert((
393 name,
394 if kind.is_dir() {
395 1
396 } else if kind.is_symlink() {
397 2
398 } else {
399 0
400 },
401 ));
402 }
403 Ok(Some(names))
404 }
405
406 fn state(&self, prefixes: Option<&[PathBuf]>) -> Result<(bool, bool)> {
410 let mut settled = true;
411 let selected = prefixes.map(|prefixes| {
412 let mut selected = BTreeSet::new();
413 for prefix in prefixes {
414 for ancestor in prefix.ancestors() {
415 if let Some(directory) = self.directories.get(ancestor) {
416 selected.insert(directory);
417 }
418 }
419 selected.extend(
420 self.directories
421 .range(prefix.clone()..)
422 .take_while(|directory| directory.starts_with(prefix)),
423 );
424 }
425 selected
426 });
427 let paths: Box<dyn Iterator<Item = (&PathBuf, bool)>> = match selected {
428 Some(selected) => Box::new(
429 selected
430 .into_iter()
431 .map(|path| (path, true))
432 .chain(self.policies.iter().map(|path| (path, false))),
433 ),
434 None => Box::new(
435 self.stamps
436 .keys()
437 .map(|path| (path, self.directories.contains(path))),
438 ),
439 };
440 for (path, directory) in paths {
441 let before = &self.stamps[path];
442 let after = membership_stamp(path)?;
443 if directory {
444 if after.is_some() && !self.directory_entries.contains_key(path) {
445 return Ok((false, false));
446 }
447 if *before != after
452 || after
453 .as_ref()
454 .is_some_and(|stamp| !stamp.membership_settled())
455 {
456 let entries = self.entries(path)?;
457 if entries.as_ref() != self.directory_entries.get(path)
458 || membership_stamp(path)? != after
459 {
460 return Ok((false, false));
461 }
462 }
463 } else {
464 let recent = after
465 .as_ref()
466 .is_some_and(|stamp| !stamp.membership_settled());
467 if (*before != after || recent)
468 && let Some(content) = self.ignore_contents.get(path)
469 {
470 if ignore_content(path, &after)?.as_ref() != Some(content) {
471 return Ok((false, false));
472 }
473 continue;
474 }
475 if *before != after {
476 return Ok((false, false));
477 }
478 settled &= !recent;
479 }
480 }
481 Ok((true, settled))
482 }
483
484 fn candidates(&self) -> Candidates {
486 Candidates {
487 files: self.files.clone(),
488 source_ignore: self.source_ignore.clone(),
489 }
490 }
491}
492
493fn git_membership_policies(root: &Path) -> BTreeSet<PathBuf> {
496 let mut policies = BTreeSet::new();
497 if let Ok(output) = paths::git(
498 root,
499 &[
500 "rev-parse",
501 "--path-format=absolute",
502 "--git-path",
503 "index",
504 "--git-path",
505 "info/exclude",
506 "--git-path",
507 "config",
508 "--git-path",
509 "config.worktree",
510 ],
511 ) && output.status.success()
512 && let Ok(paths) = String::from_utf8(output.stdout)
513 {
514 policies.extend(paths.lines().map(PathBuf::from));
515 }
516 if let Ok(output) = paths::git(root, &["config", "--null", "--show-origin", "--list"])
519 && output.status.success()
520 {
521 let output = String::from_utf8_lossy(&output.stdout);
525 let mut fields = output.split_terminator('\0');
526 while let (Some(origin), Some(entry)) = (fields.next(), fields.next()) {
527 if let Some(path) = origin.strip_prefix("file:") {
528 let origin = root.join(path);
529 if let Some((name, value)) = entry.split_once('\n')
530 && name.starts_with("include")
531 && name.ends_with(".path")
532 {
533 let included = if let Some(relative) = value.strip_prefix("~/") {
534 std::env::var_os("HOME")
535 .map(PathBuf::from)
536 .unwrap_or_default()
537 .join(relative)
538 } else {
539 origin.parent().unwrap_or(root).join(value)
540 };
541 policies.insert(included);
544 }
545 policies.insert(origin);
546 }
547 }
548 }
549 if let Ok(output) = paths::git(
550 root,
551 &["config", "--null", "--path", "--get", "core.excludesfile"],
552 ) && output.status.success()
553 && let Ok(path) = String::from_utf8(output.stdout)
554 {
555 policies.insert(root.join(path.strip_suffix('\0').unwrap_or(&path)));
556 }
557 for variable in ["GIT_CONFIG_SYSTEM", "GIT_CONFIG_GLOBAL"] {
558 if let Some(path) = std::env::var_os(variable) {
559 policies.insert(PathBuf::from(path));
560 }
561 }
562 policies.insert(PathBuf::from("/etc/gitconfig"));
563 if let Some(home) = std::env::var_os("HOME") {
564 let home = PathBuf::from(home);
565 policies.insert(home.join(".gitconfig"));
566 let config = std::env::var_os("XDG_CONFIG_HOME")
567 .map(PathBuf::from)
568 .unwrap_or_else(|| home.join(".config"));
569 policies.insert(config.join("git/config"));
570 policies.insert(config.join("git/ignore"));
571 }
572 policies
573}
574
575fn ignore_content(path: &Path, expected: &Option<Stamp>) -> Result<Option<String>> {
578 if expected.is_none() || membership_stamp(path)? != *expected {
579 return Ok(None);
580 }
581 match std::fs::symlink_metadata(path) {
582 Ok(metadata) if metadata.is_file() => {}
583 Ok(_) => return Ok(None),
584 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
585 Err(error) => return Err(error.into()),
586 }
587 let content = match std::fs::read(path) {
588 Ok(content) => content,
589 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
590 Err(error) => return Err(error.into()),
591 };
592 Ok((membership_stamp(path)? == *expected).then(|| blake3::hash(&content).to_hex().to_string()))
593}
594
595fn membership_stamp(path: &Path) -> Result<Option<Stamp>> {
597 match std::fs::symlink_metadata(path) {
598 Ok(metadata) => {
599 let mut stamp = Stamp::new(&metadata);
600 if metadata.is_symlink() {
601 match std::fs::metadata(path) {
604 Ok(target) => stamp.0.extend(Stamp::new(&target).0),
605 Err(error) if error.kind() == std::io::ErrorKind::NotFound => stamp.0[1] = -1,
606 Err(error) => return Err(error.into()),
607 }
608 }
609 Ok(Some(stamp))
610 }
611 Err(error)
612 if matches!(
613 error.kind(),
614 std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
615 ) =>
616 {
617 Ok(None)
618 }
619 Err(error) => Err(error).with_context(|| format!("cannot inspect {}", path.display())),
620 }
621}
622
623struct DependencyOutputs {
626 anchors: Vec<String>,
627 files: Vec<String>,
628 values: Mutex<HashMap<String, Value>>,
629}
630
631struct Installed {
634 content: String,
635 lockfile: Option<Lockfile>,
637 digests: Mutex<HashMap<String, String>>,
638}
639
640impl Installed {
641 fn digest(&self, key: &str, fingerprints: impl FnOnce() -> BTreeSet<String>) -> String {
642 if let Some(digest) = self.digests.lock().unwrap().get(key) {
643 return digest.clone();
644 }
645 let mut hasher = blake3::Hasher::new();
646 for fingerprint in fingerprints() {
647 hasher.update(fingerprint.as_bytes());
648 hasher.update(b"\n");
649 }
650 let digest = hasher.finalize().to_hex().to_string();
651 self.digests
652 .lock()
653 .unwrap()
654 .insert(key.to_owned(), digest.clone());
655 digest
656 }
657}
658
659impl Snapshot {
660 pub fn new(workspace: &Workspace, graph: &TaskGraph, cache_path: &Path) -> Result<Self> {
661 let canonical_root = workspace.root.canonicalize()?;
664 let (files, lockfile, (digests, workspace_prefix)) = std::thread::scope(|scope| {
665 let lockfile = scope.spawn(|| parse_lockfile(&workspace.root.join("pnpm-lock.yaml")));
666 let digests = scope.spawn(|| {
667 let digests = load_digests(&workspace.root);
668 let workspace_prefix = paths::git_path(&workspace.root, "--show-toplevel")
669 .and_then(|root| root.canonicalize().ok())
670 .and_then(|root| canonical_root.strip_prefix(root).ok().map(PathBuf::from));
671 (digests, workspace_prefix)
672 });
673 let files = source_files(&workspace.root);
674 (
675 files,
676 lockfile.join().expect("lockfile thread panicked"),
677 digests.join().expect("digest thread panicked"),
678 )
679 });
680 let mut files = files?;
681 let cache_relative = cache_path
682 .strip_prefix(&workspace.root)
683 .ok()
684 .map(PathBuf::from);
685 let mut source_exclusions: Vec<_> = graph
688 .tasks
689 .values()
690 .filter_map(|task| Outputs::new(workspace, task).ok())
691 .filter(Outputs::is_explicit)
692 .collect();
693 for task in graph.tasks.values() {
694 let Ok(entries) = crate::warm::config(workspace, task) else {
695 continue;
696 };
697 if let Ok(paths) = Outputs::from_paths(&crate::warm::kept_paths(&entries)) {
698 source_exclusions.push(paths);
699 }
700 }
701 exclude_artifacts(&mut files, &source_exclusions, cache_relative.as_deref());
702 Ok(Self {
703 files,
704 extra_candidates: BTreeSet::new(),
705 membership: Mutex::default(),
706 membership_prefixes: Mutex::default(),
707 source_exclusions,
708 cache_relative,
709 projects: ProjectGraph::build(workspace)?,
710 roots: workspace
711 .projects
712 .values()
713 .map(|project| (project.root.clone(), project.name.clone()))
714 .collect(),
715 source_ignore: Arc::new(SourceIgnore::new(&workspace.root)?),
716 generated_tasks: graph
717 .tasks
718 .values()
719 .filter_map(|task| {
720 let outputs = Outputs::new(workspace, task).ok()?;
721 outputs.is_explicit().then(|| (task.clone(), outputs))
722 })
723 .collect(),
724 task_dependencies: graph
725 .tasks
726 .iter()
727 .map(|(id, task)| (id.clone(), task.dependencies.clone()))
728 .collect(),
729 servers: graph
730 .tasks
731 .keys()
732 .map(|id| (id.clone(), servers(graph, id)))
733 .collect(),
734 canonical_root,
735 workspace_prefix,
736 patterns: Mutex::default(),
737 digests: Mutex::new(digests),
738 digests_changed: Default::default(),
739 ignored_files: Mutex::default(),
740 directories: Mutex::default(),
741 runtime: Mutex::default(),
742 lockfile: Mutex::new(lockfile),
743 dependency_outputs: Mutex::default(),
744 outputs_written: Default::default(),
745 })
746 }
747
748 fn membership_prefixes(&self, workspace: &Workspace, task: &Task) -> Result<Arc<Vec<PathBuf>>> {
750 let mut prefixes = self.membership_prefixes.lock().unwrap();
751 if let Some(prefixes) = prefixes.get(&task.id) {
752 return Ok(prefixes.clone());
753 }
754 let empty = BTreeSet::new();
755 let cancelled = AtomicBool::new(false);
756 let mut resolver = Resolver::new(self, workspace, None, &cancelled, &empty);
757 resolver.task_inputs(task)?;
758 let paths = Arc::new(
759 resolver
760 .scopes
761 .values()
762 .flat_map(|scope| &scope.patterns)
763 .filter(|pattern| !pattern.excluded)
764 .map(|pattern| &pattern.prefix)
765 .chain(resolver.json_globs.iter().map(|glob| &glob.prefix))
766 .map(|prefix| workspace.root.join(prefix))
767 .collect(),
768 );
769 prefixes.insert(task.id.clone(), Arc::clone(&paths));
770 Ok(paths)
771 }
772
773 #[cfg(test)]
776 fn current_files(&self, root: &Path) -> Result<Arc<BTreeSet<String>>> {
777 Ok(self.current_files_for(root, None, "test")?.files)
778 }
779
780 fn current_files_for(
782 &self,
783 root: &Path,
784 prefixes: Option<&[PathBuf]>,
785 task: &str,
786 ) -> Result<Candidates> {
787 let requested = Instant::now();
788 loop {
789 let previous = self.membership.lock().unwrap().clone();
790 let (unchanged, settled) = match &previous {
793 Some(entry) => entry.state(prefixes)?,
794 None => (false, false),
795 };
796 if settled && previous.as_ref().unwrap().settled {
797 return Ok(previous.as_ref().unwrap().candidates());
798 }
799 let waiting = crate::profile::span(task, "inventory_wait");
802 let mut cached = self.membership.lock().unwrap();
803 drop(waiting);
804 if let Some(entry) = &*cached
809 && entry.listed.is_some_and(|listed| listed >= requested)
810 {
811 return Ok(entry.candidates());
812 }
813 let current = match (&previous, &*cached) {
814 (Some(previous), Some(current)) => Arc::ptr_eq(previous, current),
815 (None, None) => true,
816 _ => false,
817 };
818 if !current {
819 continue;
822 }
823 drop(previous);
824 let _listing = crate::profile::span(task, "inventory_list");
825 let listed = Instant::now();
826 let rebuilt = !unchanged || settled;
831 let mut membership = if !rebuilt {
832 Arc::unwrap_or_clone(cached.take().unwrap())
833 } else {
834 Membership::new(
837 root,
838 cached.as_ref().map_or(&self.files, |entry| &entry.files),
839 &self.source_ignore,
840 &self.source_exclusions,
841 self.cache_relative.as_deref(),
842 cached
843 .as_ref()
844 .filter(|entry| unchanged && entry.settled)
845 .map(|entry| &entry.git_policies),
846 )?
847 };
848 let mut files = source_files(root)?;
849 exclude_artifacts(
850 &mut files,
851 &self.source_exclusions,
852 self.cache_relative.as_deref(),
853 );
854 let ignore = SourceIgnore::new(root)?;
855 files.extend(
856 self.extra_candidates
857 .iter()
858 .filter(|path| !ignore.matches(path))
859 .cloned(),
860 );
861 membership.source_ignore = Arc::new(ignore);
862 membership.files = Arc::new(files);
863 let candidates = membership.candidates();
864 let covered = candidates.files.iter().all(|path| {
867 Path::new(path)
868 .ancestors()
869 .skip(1)
870 .all(|parent| membership.stamps.contains_key(&root.join(parent)))
871 });
872 membership.settled = rebuilt && covered && membership.state(None)?.1;
873 membership.listed = Some(listed);
874 *cached = Some(Arc::new(membership));
875 return Ok(candidates);
876 }
877 }
878
879 pub fn save_digests(&self, root: &Path) {
881 if !self
882 .digests_changed
883 .load(std::sync::atomic::Ordering::Relaxed)
884 {
885 return;
886 }
887 let digests = self.digests.lock().unwrap();
888 let ignored = self.ignored_files.lock().unwrap();
889 let entries: BTreeMap<&String, (&Stamp, &String)> = digests
890 .iter()
891 .filter(|(path, _)| self.files.contains(*path) || ignored.contains(*path))
892 .map(|(path, (stamp, digest))| (path, (stamp, digest)))
893 .collect();
894 let path = digests_path(root);
895 let saved = (|| -> Result<()> {
896 let directory = path.parent().context("digests have a directory")?;
897 std::fs::create_dir_all(directory)?;
898 let mut file = tempfile::NamedTempFile::new_in(directory)?;
899 {
900 let mut writer = std::io::BufWriter::new(file.as_file_mut());
901 serde_json::to_writer(&mut writer, &json!({"version": 1, "entries": entries}))?;
902 writer.flush()?;
903 }
904 file.persist(&path)?;
905 Ok(())
906 })();
907 if let Err(error) = saved {
908 qk_executor::status!("qk: could not save file digests: {error:#}");
909 }
910 }
911
912 fn safe_parents(&self, root: &Path, path: &str) -> Result<()> {
914 paths::validate_path(path)?;
915 let mut current = root.to_owned();
916 for part in Path::new(path)
917 .parent()
918 .into_iter()
919 .flat_map(Path::components)
920 {
921 current.push(part);
922 if self.directories.lock().unwrap().contains(¤t) {
923 continue;
924 }
925 match std::fs::symlink_metadata(¤t) {
926 Ok(metadata) if metadata.file_type().is_symlink() => {
927 bail!("input parent is a symlink: {}", current.display())
928 }
929 Ok(metadata) if !metadata.is_dir() => {
930 bail!("input parent is not a directory: {}", current.display())
931 }
932 Ok(_) => {
933 self.directories.lock().unwrap().insert(current.clone());
934 }
935 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
936 Err(error) => return Err(error.into()),
937 }
938 }
939 Ok(())
940 }
941
942 fn installed(&self, root: &Path, task: &str) -> Result<Option<Arc<Installed>>> {
943 let absolute = root.join("pnpm-lock.yaml");
944 let Ok(metadata) = std::fs::symlink_metadata(&absolute) else {
945 return Ok(None);
946 };
947 if !metadata.is_file() {
948 return Ok(None);
949 }
950 let content = self.digest("pnpm-lock.yaml", &absolute, &metadata)?;
951 let mut current = self.lockfile.lock().unwrap();
952 if let Some(installed) = &*current
953 && installed.content == content
954 {
955 return Ok(Some(installed.clone()));
956 }
957 let parsing = crate::profile::span(task, "lockfile_parse");
958 let parsed = parse_lockfile(&absolute);
959 drop(parsing);
960 let Some(installed) = parsed else {
961 return Ok(None);
962 };
963 *current = Some(installed.clone());
964 Ok(Some(installed))
965 }
966
967 pub fn with_candidates(mut self, paths: &[String]) -> Self {
970 self.files.extend(
971 paths
972 .iter()
973 .filter(|path| !self.source_ignore.matches(path))
974 .cloned(),
975 );
976 self.extra_candidates.extend(paths.iter().cloned());
977 self
978 }
979
980 fn pattern(&self, pattern: &str, negated: bool) -> Result<Arc<Pattern>> {
981 self.compiled(pattern, negated, false)
982 }
983
984 fn fileset_pattern(&self, pattern: &str, negated: bool) -> Result<Arc<Pattern>> {
986 self.compiled(pattern, negated, true)
987 }
988
989 fn compiled(&self, pattern: &str, negated: bool, fileset: bool) -> Result<Arc<Pattern>> {
990 let key = (pattern.to_owned(), negated, fileset);
991 if let Some(pattern) = self.patterns.lock().unwrap().get(&key) {
992 return Ok(pattern.clone());
993 }
994 let compiled = Arc::new(if fileset {
995 Pattern::fileset(pattern, negated)?
996 } else {
997 Pattern::new(pattern, negated)?
998 });
999 self.patterns.lock().unwrap().insert(key, compiled.clone());
1000 Ok(compiled)
1001 }
1002
1003 fn digest(&self, path: &str, absolute: &Path, metadata: &std::fs::Metadata) -> Result<String> {
1004 let stamp = Stamp::new(metadata);
1005 if let Some((known, digest)) = self.digests.lock().unwrap().get(path)
1006 && *known == stamp
1007 {
1008 return Ok(digest.clone());
1009 }
1010 let digest = digest_file(absolute)?;
1011 let after = Stamp::new(&std::fs::symlink_metadata(absolute)?);
1013 if after == stamp && stamp.settled() {
1014 self.digests
1015 .lock()
1016 .unwrap()
1017 .insert(path.to_owned(), (stamp, digest.clone()));
1018 self.digests_changed
1019 .store(true, std::sync::atomic::Ordering::Relaxed);
1020 }
1021 Ok(digest)
1022 }
1023
1024 fn dependency_outputs(
1027 &self,
1028 workspace: &Workspace,
1029 task: &Task,
1030 ) -> Result<Arc<DependencyOutputs>> {
1031 if let Some(outputs) = self.dependency_outputs.lock().unwrap().get(&task.id) {
1032 return Ok(outputs.clone());
1033 }
1034 let written = self
1035 .outputs_written
1036 .load(std::sync::atomic::Ordering::SeqCst);
1037 let outputs = Arc::new(self.fresh_dependency_outputs(workspace, task)?);
1038 let mut listed = self.dependency_outputs.lock().unwrap();
1039 if self
1040 .outputs_written
1041 .load(std::sync::atomic::Ordering::SeqCst)
1042 != written
1043 {
1044 return Ok(outputs);
1045 }
1046 Ok(listed.entry(task.id.clone()).or_insert(outputs).clone())
1047 }
1048
1049 fn fresh_dependency_outputs(
1051 &self,
1052 workspace: &Workspace,
1053 task: &Task,
1054 ) -> Result<DependencyOutputs> {
1055 let declared = Outputs::new(workspace, task)?;
1056 let mut files = Vec::new();
1057 for entry in declared.entries(&workspace.root)? {
1058 let (path, metadata) = entry?;
1059 if !metadata.is_dir() {
1060 files.push(path);
1061 }
1062 }
1063 Ok(DependencyOutputs {
1064 anchors: declared.anchors().map(str::to_owned).collect(),
1065 files,
1066 values: Mutex::default(),
1067 })
1068 }
1069
1070 pub(crate) fn outputs_written(&self, workspace: &Workspace, task: &Task) {
1073 let anchors: Option<Vec<String>> = Outputs::new(workspace, task)
1074 .ok()
1075 .filter(Outputs::is_explicit)
1076 .map(|outputs| outputs.anchors().map(str::to_owned).collect());
1077 let overlap = |a: &str, b: &str| {
1078 a == b
1079 || b.strip_prefix(a).is_some_and(|rest| rest.starts_with('/'))
1080 || a.strip_prefix(b).is_some_and(|rest| rest.starts_with('/'))
1081 };
1082 let mut listed = self.dependency_outputs.lock().unwrap();
1083 self.outputs_written
1084 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1085 listed.retain(|_, outputs| {
1086 anchors.as_ref().is_some_and(|written| {
1087 !written
1088 .iter()
1089 .any(|a| outputs.anchors.iter().any(|b| overlap(a, b)))
1090 })
1091 });
1092 }
1093
1094 fn ignored_value(&self, root: &Path, path: &str) -> Result<Option<Value>> {
1097 let absolute = root.join(path);
1098 let metadata = match std::fs::metadata(&absolute) {
1099 Ok(metadata) if metadata.is_file() => metadata,
1100 Ok(_) => return Ok(None),
1101 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1102 Err(error) => return Err(error.into()),
1103 };
1104 let mode = crate::store::mode(&metadata) & 0o111 != 0;
1105 self.ignored_files.lock().unwrap().insert(path.to_owned());
1106 Ok(Some(
1107 json!({"content":self.digest(path, &absolute, &metadata)?, "mode":mode}),
1108 ))
1109 }
1110
1111 pub(crate) fn recorded_value(&self, root: &Path, path: &str) -> Result<Value> {
1113 self.file_value(root, path)
1114 }
1115
1116 fn file_value(&self, root: &Path, path: &str) -> Result<Value> {
1117 self.file_value_with_links(root, path, &mut BTreeSet::new())
1118 }
1119
1120 fn file_value_with_links(
1121 &self,
1122 root: &Path,
1123 path: &str,
1124 visiting: &mut BTreeSet<PathBuf>,
1125 ) -> Result<Value> {
1126 self.safe_parents(root, path)?;
1127 let absolute = root.join(path);
1128 let metadata = std::fs::symlink_metadata(&absolute)?;
1129 if metadata.file_type().is_symlink() {
1130 let target = std::fs::read_link(&absolute)?;
1131 let Ok(resolved) = absolute.canonicalize() else {
1133 return Ok(json!({"link":target, "dangling":true}));
1134 };
1135 let Ok(relative) = resolved.strip_prefix(&self.canonical_root) else {
1136 bail!("input symlink resolves outside the workspace: {path}");
1137 };
1138 if resolved.is_file() {
1139 let metadata = std::fs::metadata(&resolved)?;
1140 let key = paths::relative(Path::new(""), relative)?;
1141 return Ok(
1142 json!({"link":target, "content":self.digest(&key, &resolved, &metadata)?,
1143 "mode":crate::store::mode(&metadata) & 0o111 != 0}),
1144 );
1145 }
1146 let relative = paths::relative(Path::new(""), relative)?;
1147 if relative.is_empty() {
1148 bail!("input symlink resolves to the workspace root: {path}");
1149 }
1150 if !visiting.insert(resolved.clone()) {
1151 bail!("directory input symlink cycle: {path}");
1152 }
1153 let mut contents = BTreeMap::new();
1154 for entry in walkdir::WalkDir::new(&resolved).follow_links(false) {
1157 let entry = entry?;
1158 if entry.file_type().is_dir() {
1159 continue;
1160 }
1161 let file = paths::relative(&self.canonical_root, entry.path())?;
1162 let value = self.file_value_with_links(&self.canonical_root, &file, visiting)?;
1163 contents.insert(
1164 file.strip_prefix(&relative).unwrap_or(&file).to_owned(),
1165 value,
1166 );
1167 }
1168 visiting.remove(&resolved);
1169 return Ok(json!({"link":target, "directory":contents}));
1170 }
1171 if !metadata.is_file() {
1172 bail!("input is not a regular file: {path}");
1173 }
1174 let mode = crate::store::mode(&metadata) & 0o111 != 0;
1177 Ok(json!({"content":self.digest(path, &absolute, &metadata)?, "mode":mode}))
1178 }
1179}
1180
1181fn parse_lockfile(path: &Path) -> Option<Arc<Installed>> {
1184 let text = std::fs::read_to_string(path).ok()?;
1185 let lockfile = match Lockfile::parse(&text) {
1186 Ok(lockfile) => Some(lockfile),
1187 Err(error) => {
1188 qk_executor::status!("qk: pnpm-lock.yaml: keying tasks by the whole file ({error:#})");
1189 None
1190 }
1191 };
1192 Some(Arc::new(Installed {
1193 content: blake3::hash(text.as_bytes()).to_hex().to_string(),
1194 lockfile,
1195 digests: Mutex::default(),
1196 }))
1197}
1198
1199fn load_digests(root: &Path) -> HashMap<String, (Stamp, String)> {
1200 #[derive(serde::Deserialize)]
1201 struct Saved {
1202 version: u32,
1203 entries: HashMap<String, (Stamp, String)>,
1204 }
1205 std::fs::read(digests_path(root))
1206 .ok()
1207 .and_then(|bytes| serde_json::from_slice::<Saved>(&bytes).ok())
1208 .filter(|saved| saved.version == 1)
1209 .map(|saved| saved.entries)
1210 .unwrap_or_default()
1211}
1212
1213fn exclude_artifacts(files: &mut BTreeSet<String>, outputs: &[Outputs], cache: Option<&Path>) {
1216 let mut generated = BTreeSet::new();
1217 for outputs in outputs {
1218 for anchor in outputs.anchors() {
1219 generated.extend(
1220 under(files, anchor)
1221 .filter(|path| outputs.matches(path))
1222 .cloned(),
1223 );
1224 }
1225 }
1226 files.retain(|path| {
1227 !generated.contains(path) && cache.is_none_or(|cache| !Path::new(path).starts_with(cache))
1228 });
1229}
1230
1231impl Snapshot {
1233 fn owner(&self, path: &str) -> Option<&str> {
1236 let mut path = path;
1237 loop {
1238 if let Some(project) = self.roots.get(path) {
1239 return Some(project);
1240 }
1241 match path.rfind('/') {
1242 Some(slash) => path = &path[..slash],
1243 None if path != "." => path = ".",
1244 None => return None,
1245 }
1246 }
1247 }
1248}
1249
1250fn under<'a>(files: &'a BTreeSet<String>, prefix: &'a str) -> impl Iterator<Item = &'a String> {
1251 files
1252 .range::<str, _>((
1253 std::ops::Bound::Included(prefix),
1254 std::ops::Bound::Unbounded,
1255 ))
1256 .take_while(move |path| path.starts_with(prefix))
1257 .filter(move |path| path.len() == prefix.len() || path.as_bytes()[prefix.len()] == b'/')
1258}
1259
1260pub struct SourceIgnore(ignore::gitignore::Gitignore);
1262
1263impl SourceIgnore {
1264 pub fn new(root: &Path) -> Result<Self> {
1266 let mut builder = ignore::gitignore::GitignoreBuilder::new(root);
1267 for name in [".gitignore", ".nxignore"] {
1268 let path = root.join(name);
1269 if path.is_file()
1270 && let Some(error) = builder.add(path)
1271 {
1272 return Err(error.into());
1273 }
1274 }
1275 Ok(Self(builder.build()?))
1276 }
1277
1278 fn matches_directory(&self, path: &str) -> bool {
1279 self.matches(path) || self.0.matched(path, true).is_ignore()
1280 }
1281
1282 pub fn matches(&self, path: &str) -> bool {
1284 Path::new(path)
1286 .ancestors()
1287 .skip(1)
1288 .take_while(|parent| !parent.as_os_str().is_empty())
1289 .any(|parent| self.0.matched(parent, true).is_ignore())
1290 || self.0.matched(path, false).is_ignore()
1291 }
1292}
1293
1294pub fn source_files(root: &Path) -> Result<BTreeSet<String>> {
1296 let lines =
1297 |output: std::io::Result<std::process::Output>| -> Option<Result<BTreeSet<String>>> {
1298 let output = output.ok().filter(|output| output.status.success())?;
1299 Some(
1300 output
1301 .stdout
1302 .split(|byte| *byte == 0)
1303 .filter(|path| !path.is_empty())
1304 .map(|path| {
1305 String::from_utf8(path.to_vec()).context("input paths must be UTF-8")
1306 })
1307 .collect(),
1308 )
1309 };
1310 let (listed, deleted) = if root.join(".nxignore").is_file() {
1315 (None, None)
1318 } else {
1319 std::thread::scope(|scope| {
1320 let deleted =
1321 scope.spawn(|| paths::git(root, &["ls-files", "--deleted", "-z", "--", "."]));
1322 let listed = paths::git(
1323 root,
1324 &[
1325 "ls-files",
1326 "-v",
1327 "--cached",
1328 "--others",
1329 "--exclude-standard",
1330 "-z",
1331 "--",
1332 ".",
1333 ],
1334 );
1335 (
1336 lines(listed),
1337 lines(deleted.join().expect("git listing thread panicked")),
1338 )
1339 })
1340 };
1341 let candidates = match (listed, deleted) {
1342 (Some(listed), Some(deleted)) => {
1343 let deleted = deleted?;
1344 let mut files = BTreeSet::new();
1345 for entry in listed? {
1346 let Some((tag, path)) = entry.split_once(' ') else {
1347 bail!("unexpected git ls-files entry {entry:?}");
1348 };
1349 let absent =
1350 matches!(tag, "S" | "s") && std::fs::symlink_metadata(root.join(path)).is_err();
1351 if !absent && !deleted.contains(path) {
1352 files.insert(path.to_owned());
1353 }
1354 }
1355 files
1356 }
1357 _ => {
1358 let mut files = BTreeSet::new();
1359 for entry in ignore::WalkBuilder::new(root)
1360 .hidden(false)
1361 .parents(false)
1362 .git_global(false)
1363 .require_git(false)
1364 .ignore(false)
1365 .add_custom_ignore_filename(".nxignore")
1366 .follow_links(false)
1367 .filter_entry(|entry| {
1368 !matches!(
1369 entry.file_name().to_str(),
1370 Some(".git" | ".qk" | "node_modules")
1371 )
1372 })
1373 .build()
1374 {
1375 let entry = entry?;
1376 if entry.file_type().is_some_and(|kind| !kind.is_dir()) {
1377 files.insert(paths::relative(root, entry.path())?);
1378 }
1379 }
1380 files
1381 }
1382 };
1383 let ignore = SourceIgnore::new(root)?;
1384 Ok(candidates
1385 .into_iter()
1386 .filter(|path| {
1387 !ignore.matches(path)
1388 && !path
1389 .split('/')
1390 .any(|part| matches!(part, ".git" | ".qk" | "node_modules"))
1391 })
1392 .collect())
1393}
1394
1395pub(crate) fn json_projection(path: &str, text: &str, fields: &JsonFields) -> Option<Value> {
1399 let mut value: Value = if path.ends_with(".yaml") || path.ends_with(".yml") {
1400 serde_yaml_ng::from_str(text).ok()?
1401 } else {
1402 jsonc_parser::parse_to_serde_value(text, &Default::default()).ok()?
1403 };
1404 if let Some(fields) = &fields.fields {
1405 let mut kept = json!({});
1406 for field in fields {
1407 if let Some(found) = json_path(&value, field) {
1408 set_json_path(&mut kept, field, found.clone());
1409 }
1410 }
1411 value = kept;
1412 }
1413 for field in &fields.exclude {
1414 remove_json_path(&mut value, field);
1415 }
1416 Some(value)
1417}
1418
1419fn json_path<'v>(value: &'v Value, path: &str) -> Option<&'v Value> {
1420 path.split('.')
1421 .try_fold(value, |value, part| value.get(part))
1422}
1423
1424fn set_json_path(target: &mut Value, path: &str, value: Value) {
1425 let mut parts = path.split('.').peekable();
1426 let mut current = target;
1427 while let Some(part) = parts.next() {
1428 let Value::Object(object) = current else {
1429 return;
1430 };
1431 if parts.peek().is_none() {
1432 object.insert(part.to_owned(), value);
1433 return;
1434 }
1435 current = object.entry(part).or_insert_with(|| json!({}));
1436 }
1437}
1438
1439fn remove_json_path(target: &mut Value, path: &str) {
1440 let (parent, last) = match path.rsplit_once('.') {
1441 Some((parent, last)) => (Some(parent), last),
1442 None => (None, path),
1443 };
1444 let parent = match parent {
1445 Some(parent) => parent
1446 .split('.')
1447 .try_fold(&mut *target, |value, part| value.get_mut(part)),
1448 None => Some(target),
1449 };
1450 if let Some(Value::Object(object)) = parent {
1451 object.remove(last);
1452 }
1453}
1454
1455struct FilePattern {
1456 matcher: Arc<Pattern>,
1457 prefix: String,
1458 rest: Option<String>,
1461 excluded: bool,
1462 owner: Option<String>,
1464}
1465
1466#[derive(Default)]
1470pub struct Selection {
1471 scopes: Vec<Vec<FilePattern>>,
1472}
1473
1474impl Selection {
1475 fn selects(&self, snapshot: &Snapshot, path: &str) -> bool {
1479 let applies = |pattern: &FilePattern| {
1480 pattern.matcher.is_match(path)
1481 && (pattern.owner.is_none() || snapshot.owner(path) == pattern.owner.as_deref())
1482 };
1483 self.scopes.iter().any(|patterns| {
1484 patterns
1485 .iter()
1486 .any(|pattern| !pattern.excluded && applies(pattern))
1487 && !patterns
1488 .iter()
1489 .any(|pattern| pattern.excluded && applies(pattern))
1490 })
1491 }
1492
1493 fn selects_every_entry(&self, snapshot: &Snapshot, directory: &str) -> bool {
1498 let within = |prefix: &str| {
1499 prefix.is_empty()
1500 || directory == prefix
1501 || directory
1502 .strip_prefix(prefix)
1503 .is_some_and(|rest| rest.starts_with('/'))
1504 };
1505 let reaches = |prefix: &str| {
1506 within(prefix)
1507 || directory.is_empty()
1508 || prefix
1509 .strip_prefix(directory)
1510 .is_some_and(|rest| rest.starts_with('/'))
1511 };
1512 let owner = snapshot.owner(if directory.is_empty() { "." } else { directory });
1513 self.scopes.iter().any(|patterns| {
1514 patterns.iter().any(|pattern| {
1515 !pattern.excluded
1516 && within(&pattern.prefix)
1517 && matches!(pattern.rest.as_deref(), Some("**" | "**/*"))
1518 && (pattern.owner.is_none() || owner == pattern.owner.as_deref())
1519 }) && !patterns
1520 .iter()
1521 .any(|pattern| pattern.excluded && reaches(&pattern.prefix))
1522 })
1523 }
1524}
1525
1526pub(crate) struct Coverage {
1533 resolved: Resolved,
1534 dependency_outputs: Vec<(Arc<Pattern>, Vec<Outputs>)>,
1537}
1538
1539pub(crate) fn coverage(
1541 snapshot: &Snapshot,
1542 workspace: &Workspace,
1543 graph: &TaskGraph,
1544 task: &Task,
1545 prepared: &PreparedTask,
1546 cancelled: &AtomicBool,
1547) -> Result<Coverage> {
1548 let resolved = resolve(snapshot, workspace, task, Some(prepared), cancelled)?;
1549 let selections: Vec<(String, bool)> = resolved
1550 .dependency_output_inputs()
1551 .map(|(pattern, transitive)| (pattern.to_owned(), transitive))
1552 .collect();
1553 let mut dependency_outputs = Vec::new();
1554 for (pattern, transitive) in selections {
1555 let matcher = snapshot.pattern(&pattern, false)?;
1556 let mut selected = task.dependencies.clone();
1557 if transitive {
1558 let mut pending = selected.clone();
1559 while let Some(id) = pending.pop_first() {
1560 for dependency in &graph.tasks[&id].dependencies {
1561 if selected.insert(dependency.clone()) {
1562 pending.insert(dependency.clone());
1563 }
1564 }
1565 }
1566 }
1567 let mut outputs = Vec::new();
1568 for id in selected {
1569 let declared = Outputs::new(workspace, &graph.tasks[&id])?;
1570 if declared.declared() {
1571 outputs.push(declared);
1572 }
1573 }
1574 dependency_outputs.push((matcher, outputs));
1575 }
1576 Ok(Coverage {
1577 resolved,
1578 dependency_outputs,
1579 })
1580}
1581
1582impl Coverage {
1583 fn keyed(&self, path: &str) -> bool {
1586 let inferred = &self.resolved.inferred;
1587 self.resolved.is_whole_file(path)
1588 || self.resolved.is_json_input(path)
1589 || inferred.manifests.contains_key(path)
1590 || inferred.tsconfig.as_deref() == Some(path)
1591 || inferred.settings.contains(path)
1592 || inferred.project.contains(path)
1593 || (inferred.workspace_file && path == "pnpm-workspace.yaml")
1594 || (inferred.lockfile.is_some() && path == "pnpm-lock.yaml")
1595 || self.dependency_outputs.iter().any(|(matcher, outputs)| {
1596 matcher.is_match(path) && outputs.iter().any(|outputs| outputs.matches(path))
1597 })
1598 }
1599
1600 pub(crate) fn covers(
1610 &self,
1611 snapshot: &Snapshot,
1612 root: &Path,
1613 set: &crate::recorded::PathSet,
1614 output: &dyn Fn(&str) -> bool,
1615 ) -> bool {
1616 let selection = &self.resolved.declared.selection;
1617 fn directory(path: &str) -> &str {
1618 if path == "." { "" } else { path }
1619 }
1620 let holds_keyed = |path: &str| {
1621 let path = directory(path);
1622 if path.is_empty() {
1623 return self.resolved.whole_files().next().is_some();
1624 }
1625 under(&self.resolved.declared.files, path).next().is_some()
1626 || under(&self.resolved.inferred.files, path).next().is_some()
1627 };
1628 set.reads.iter().all(|path| self.keyed(path))
1629 && set
1630 .present
1631 .iter()
1632 .all(|path| self.keyed(path) || (root.join(path).is_dir() && holds_keyed(path)))
1633 && set.absent.iter().all(|path| {
1634 self.keyed(path)
1635 || (!snapshot.source_ignore.matches(path)
1636 && !output(path)
1637 && selection.selects(snapshot, path))
1638 })
1639 && set.listings.iter().all(|path| {
1640 let path = directory(path);
1641 (path.is_empty() || !output(path)) && selection.selects_every_entry(snapshot, path)
1642 })
1643 }
1644}
1645
1646struct JsonGlob {
1648 matcher: Arc<Pattern>,
1649 prefix: String,
1650 owner: Option<String>,
1652 fields: JsonFields,
1653}
1654
1655#[derive(Default)]
1656struct FileSelection {
1657 patterns: Vec<FilePattern>,
1658 included: BTreeSet<String>,
1659 excluded: BTreeSet<String>,
1660}
1661
1662#[derive(PartialEq, Eq, PartialOrd, Ord)]
1663enum FileScope {
1664 Workspace,
1665 Project(String),
1666}
1667
1668struct Resolver<'a> {
1669 workspace: &'a Workspace,
1670 snapshot: &'a Snapshot,
1671 candidates: &'a BTreeSet<String>,
1672 selected: BTreeSet<String>,
1673 scopes: BTreeMap<FileScope, FileSelection>,
1674 values: BTreeMap<String, Value>,
1675 json_fields: BTreeMap<String, Vec<JsonFields>>,
1676 json_globs: Vec<JsonGlob>,
1677 named_stack: Vec<(String, String)>,
1678 expanded: BTreeSet<(String, String)>,
1680 external: BTreeSet<String>,
1681 ignored: BTreeMap<(String, bool), Vec<String>>,
1685 external_declared: bool,
1686 projects: BTreeSet<String>,
1689 prepared: Option<&'a PreparedTask>,
1692 cancelled: &'a AtomicBool,
1693}
1694
1695impl<'a> Resolver<'a> {
1696 fn new(
1697 snapshot: &'a Snapshot,
1698 workspace: &'a Workspace,
1699 prepared: Option<&'a PreparedTask>,
1700 cancelled: &'a AtomicBool,
1701 candidates: &'a BTreeSet<String>,
1702 ) -> Self {
1703 Self {
1704 workspace,
1705 snapshot,
1706 candidates,
1707 selected: BTreeSet::new(),
1708 scopes: BTreeMap::new(),
1709 values: BTreeMap::new(),
1710 json_fields: BTreeMap::new(),
1711 json_globs: Vec::new(),
1712 named_stack: Vec::new(),
1713 expanded: BTreeSet::new(),
1714 external: BTreeSet::new(),
1715 ignored: BTreeMap::new(),
1716 external_declared: false,
1717 projects: BTreeSet::new(),
1718 prepared,
1719 cancelled,
1720 }
1721 }
1722
1723 fn task_inputs(&mut self, task: &Task) -> Result<()> {
1724 self.projects.insert(task.project.clone());
1725 for input in declared_inputs(task) {
1726 self.input(&task.project, &input)?;
1727 }
1728 let snapshot = self.snapshot;
1731 for (project, inputs) in snapshot.servers.get(&task.id).into_iter().flatten() {
1732 for input in inputs {
1733 self.input(project, input)
1734 .with_context(|| format!("inputs of {project}, which serves this task"))?;
1735 }
1736 }
1737 Ok(())
1738 }
1739
1740 fn input(&mut self, project: &str, input: &Value) -> Result<()> {
1741 self.projects.insert(project.to_owned());
1742 if let Value::Object(object) = input
1745 && let Some(always) = object.get("always")
1746 {
1747 if always.as_bool().context("input always must be boolean")?
1750 && let Some(Value::String(name)) = object.get("input")
1751 && let Some(projects) = object.get("projects")
1752 {
1753 let patterns: Vec<String> = match projects {
1754 Value::String(pattern) => vec![pattern.clone()],
1755 other => serde_json::from_value(other.clone())
1756 .context("input projects must be a string or an array")?,
1757 };
1758 for pattern in patterns.iter().filter(|pattern| {
1759 !pattern.starts_with('!')
1760 && !matches!(pattern.as_str(), "self" | "dependencies")
1761 }) {
1762 if qk_graph::select_projects(
1763 &self.workspace.projects,
1764 std::slice::from_ref(pattern),
1765 &[],
1766 )?
1767 .is_empty()
1768 {
1769 bail!(
1770 "the input {{\"input\": \"{name}\", \"projects\"}} with \"always\": true selects \"{pattern}\", which matches no project"
1771 );
1772 }
1773 }
1774 }
1775 let mut object = object.clone();
1776 object.remove("always");
1777 return self.input(project, &Value::Object(object));
1778 }
1779 match input {
1780 Value::String(name)
1784 if name.starts_with("^{projectRoot}") || name.starts_with("^{workspaceRoot}") =>
1785 {
1786 self.dependency_fileset(project, &name[1..])?;
1787 }
1788 Value::String(name) if name.starts_with('^') => {
1789 self.dependencies_named(project, &name[1..])?;
1790 }
1791 Value::Object(object)
1793 if object.contains_key("input") && !object.contains_key("fileset") =>
1794 {
1795 let name = object["input"]
1796 .as_str()
1797 .context("input must name a named input")?;
1798 let projects = object.get("projects");
1799 let dependencies = object.get("dependencies") == Some(&json!(true))
1800 || projects == Some(&json!("dependencies"));
1801 let listed = projects.filter(|projects| {
1802 **projects != json!("self") && **projects != json!("dependencies")
1803 });
1804 if (dependencies || listed.is_some()) && !self.named_stack.is_empty() {
1805 bail!("named inputs can only refer to named inputs of their own project");
1806 }
1807 if let Some(listed) = listed {
1808 let patterns: Vec<String> = match listed {
1809 Value::String(pattern) => vec![pattern.clone()],
1810 other => serde_json::from_value(other.clone())
1811 .context("input projects must be a string or an array")?,
1812 };
1813 for selected in qk_config::find_matching_projects(
1816 &patterns,
1817 &self.workspace.projects,
1818 qk_config::Reader::Native,
1819 ) {
1820 if dependencies {
1823 self.dependencies_named(&selected, name)?;
1824 } else if self.expanded.insert((selected.clone(), name.to_owned())) {
1825 self.named(&selected, name)?;
1826 }
1827 }
1828 } else if dependencies {
1829 self.dependencies_named(project, name)?;
1830 } else {
1831 self.named(project, name)?;
1832 }
1833 }
1834 Value::Object(object) if object.contains_key("fileset") => {
1835 if let Some(key) = object.keys().find(|key| {
1836 !matches!(key.as_str(), "fileset" | "dependencies" | "includeIgnored")
1837 }) {
1838 bail!("unsupported fileset input field {key:?}");
1839 }
1840 let flag = |name: &str| {
1841 object
1842 .get(name)
1843 .map(|value| {
1844 value
1845 .as_bool()
1846 .with_context(|| format!("{name} must be boolean"))
1847 })
1848 .transpose()
1849 .map(Option::unwrap_or_default)
1850 };
1851 let (dependencies, include_ignored) =
1852 (flag("dependencies")?, flag("includeIgnored")?);
1853 let pattern = object["fileset"]
1854 .as_str()
1855 .context("fileset input must be a glob")?
1856 .to_owned();
1857 validate_fileset(&pattern)?;
1858 match (dependencies, include_ignored) {
1859 (false, false) => self.fileset(project, &pattern)?,
1860 (true, false) => self.dependency_fileset(project, &pattern)?,
1861 (false, true) => self.ignored_fileset(project, &pattern, false),
1862 (true, true) => {
1863 if !self.named_stack.is_empty() {
1864 bail!(
1865 "named inputs can only refer to named inputs of their own project"
1866 );
1867 }
1868 for dependency in self.dependency_closure(project) {
1869 self.ignored_fileset(&dependency, &pattern, true);
1870 }
1871 }
1872 }
1873 }
1874 Value::Object(object)
1875 if object.len() == 1 && object.contains_key("workingDirectory") =>
1876 {
1877 let mode = object["workingDirectory"].as_str().unwrap_or_default();
1878 let directory = std::env::current_dir()?;
1879 let value = match mode {
1880 "absolute" => directory.to_string_lossy().into_owned(),
1881 "relative" => {
1882 let root = self.workspace.root.canonicalize()?;
1883 let directory = directory.canonicalize()?;
1884 paths::relative(&root, &directory)
1885 .unwrap_or_else(|_| directory.to_string_lossy().into_owned())
1886 }
1887 _ => bail!("workingDirectory must be \"relative\" or \"absolute\""),
1888 };
1889 self.values
1890 .insert(format!("workingDirectory:{mode}"), json!(value));
1891 }
1892 Value::Object(object) if object.contains_key("json") => {
1893 self.json(project, object)?;
1894 }
1895 Value::String(name) if self.has_named(project, name) => self.named(project, name)?,
1898 Value::String(pattern) => {
1899 validate_fileset(pattern)?;
1900 self.fileset(project, pattern)?
1901 }
1902 Value::Object(object) if object.len() == 1 && object.contains_key("env") => {
1903 let name = object["env"]
1904 .as_str()
1905 .context("env input must name a variable")?;
1906 let Some(prepared) = self.prepared else {
1907 self.values.insert(format!("env:{name}"), Value::Null);
1908 return Ok(());
1909 };
1910 let value = prepared
1911 .env
1912 .get(std::ffi::OsStr::new(name))
1913 .map(|value| {
1914 value
1915 .to_str()
1916 .map(str::to_owned)
1917 .context("declared env input must be UTF-8")
1918 })
1919 .transpose()?;
1920 self.values.insert(format!("env:{name}"), json!(value));
1921 }
1922 Value::Object(object) if object.len() == 1 && object.contains_key("runtime") => {
1923 let command = object["runtime"]
1924 .as_str()
1925 .context("runtime input must be a command")?;
1926 let Some(prepared) = self.prepared else {
1927 self.values
1928 .insert(format!("runtime:{command}"), Value::Null);
1929 return Ok(());
1930 };
1931 let memo = RuntimeKey {
1932 command: command.into(),
1933 environment: prepared.runtime_env.clone(),
1934 };
1935 let slot = self
1936 .snapshot
1937 .runtime
1938 .lock()
1939 .unwrap()
1940 .entry(memo)
1941 .or_default()
1942 .clone();
1943 let mut saved = slot.lock().unwrap();
1944 if let Some(value) = saved.as_ref() {
1945 self.values
1946 .insert(format!("runtime:{command}"), value.clone());
1947 return Ok(());
1948 }
1949 let _profile = crate::profile::span(command, "runtime");
1950 let mut prepared = prepared.clone();
1951 prepared.env = prepared.runtime_env.clone();
1952 prepared.execution.clear();
1953 prepared.node_path = None;
1954 prepared.commands = vec![command.into()];
1955 prepared.cwd = self.workspace.root.clone();
1956 prepared.parallel = false;
1957 prepared.pseudo_terminal = false;
1959 let log = tempfile::NamedTempFile::new()?;
1960 let capture = Capture::new(Some(log.as_file().try_clone()?), Display::Hidden);
1961 if execute_captured(&prepared, self.cancelled, Some(&capture))? != Outcome::Success
1962 {
1963 bail!("runtime input did not succeed");
1964 }
1965 let mut stdout = blake3::Hasher::new();
1966 let mut stderr = blake3::Hasher::new();
1967 read_capture(File::open(log.path())?, |error, bytes| {
1968 if error {
1969 stderr.update(bytes);
1970 } else {
1971 stdout.update(bytes);
1972 }
1973 Ok(())
1974 })?;
1975 let value = json!([
1976 stdout.finalize().to_hex().to_string(),
1977 stderr.finalize().to_hex().to_string()
1978 ]);
1979 *saved = Some(value.clone());
1980 self.values.insert(format!("runtime:{command}"), value);
1981 }
1982 Value::Object(object) if object.contains_key("dependentTasksOutputFiles") => {
1983 if object
1984 .keys()
1985 .any(|key| !matches!(key.as_str(), "dependentTasksOutputFiles" | "transitive"))
1986 {
1987 bail!("unsupported output input field");
1988 }
1989 let pattern = object["dependentTasksOutputFiles"]
1990 .as_str()
1991 .context("dependentTasksOutputFiles must be a glob")?;
1992 self.snapshot.pattern(pattern, false)?;
1993 let transitive = object
1994 .get("transitive")
1995 .map(|value| value.as_bool().context("transitive must be boolean"))
1996 .transpose()?
1997 .unwrap_or(false);
1998 let value = self
2002 .values
2003 .entry(format!("dependentTasksOutputFiles:{pattern}"))
2004 .or_insert(json!(false));
2005 if transitive {
2006 *value = json!(true);
2007 }
2008 }
2009 Value::Object(object)
2010 if object.len() == 1 && object.contains_key("externalDependencies") =>
2011 {
2012 let names: Vec<String> =
2013 serde_json::from_value(object["externalDependencies"].clone())?;
2014 if let Some(name) = names
2016 .iter()
2017 .find(|name| self.workspace.projects.contains_key(*name))
2018 {
2019 if self.snapshot.projects.dependencies[project]
2020 .iter()
2021 .any(|edge| edge.target == *name)
2022 {
2023 bail!(
2024 "the externalDependency {name:?} of {project} is not an external node and is already a dependency; remove it from the externalDependencies inputs"
2025 );
2026 }
2027 bail!(
2028 "the externalDependency {name:?} of {project} is not an external node; if it is a dependency, add it to the project's implicitDependencies"
2029 );
2030 }
2031 self.external.extend(names);
2032 self.external_declared = true;
2033 }
2034 _ => bail!("unsupported task input declaration"),
2035 }
2036 Ok(())
2037 }
2038
2039 fn fileset(&mut self, project: &str, pattern: &str) -> Result<()> {
2040 self.projects.insert(project.to_owned());
2041 let (exclude, pattern) = pattern
2042 .strip_prefix('!')
2043 .map(|pattern| (true, pattern))
2044 .unwrap_or((false, pattern));
2045 let (scope, owner) = if pattern.starts_with("{projectRoot}/") {
2051 (
2052 FileScope::Project(project.to_owned()),
2053 Some(project.to_owned()),
2054 )
2055 } else if pattern.starts_with("{workspaceRoot}/") {
2056 (FileScope::Workspace, None)
2057 } else {
2058 return Ok(());
2059 };
2060 let pattern = paths::expand_glob(self.workspace, project, pattern)?;
2061 let matcher = self.snapshot.fileset_pattern(&pattern, exclude)?;
2062 let (prefix, rest) = crate::ignored::partition(pattern.trim_end_matches('/'));
2063 let rest = rest.map(str::to_owned);
2064 let candidates: Box<dyn Iterator<Item = &String>> = if prefix.is_empty() {
2065 Box::new(self.candidates.iter())
2066 } else {
2067 Box::new(under(self.candidates, &prefix))
2068 };
2069 let snapshot = self.snapshot;
2070 let selection = self.scopes.entry(scope).or_default();
2071 let selected = if exclude {
2072 &mut selection.excluded
2073 } else {
2074 &mut selection.included
2075 };
2076 selected.extend(
2077 candidates
2078 .filter(|file| matcher.is_match(file))
2079 .filter(|file| owner.is_none() || snapshot.owner(file) == owner.as_deref())
2080 .cloned(),
2081 );
2082 selection.patterns.push(FilePattern {
2083 matcher,
2084 prefix,
2085 rest,
2086 excluded: exclude,
2087 owner,
2088 });
2089 Ok(())
2090 }
2091
2092 fn ignored_fileset(&mut self, project: &str, pattern: &str, dependencies: bool) {
2095 let root = &self.workspace.projects[project].root;
2098 let resolved = if root == "." {
2099 pattern.replace("{projectRoot}/", "")
2100 } else {
2101 pattern.replace("{projectRoot}", &crate::glob::escape(root))
2102 }
2103 .replace("{projectName}", &crate::glob::escape(project));
2104 let resolved = match resolved.strip_prefix("!{workspaceRoot}/") {
2105 Some(rest) => format!("!{rest}"),
2106 None => resolved
2107 .strip_prefix("{workspaceRoot}/")
2108 .map(str::to_owned)
2109 .unwrap_or(resolved),
2110 };
2111 self.ignored
2112 .entry((project.to_owned(), dependencies))
2113 .or_default()
2114 .push(resolved);
2115 }
2116
2117 fn dependency_fileset(&mut self, project: &str, pattern: &str) -> Result<()> {
2119 if !self.named_stack.is_empty() {
2120 bail!("named inputs can only refer to named inputs of their own project");
2121 }
2122 for dependency in self.dependency_closure(project) {
2123 self.fileset(&dependency, pattern)?;
2124 }
2125 Ok(())
2126 }
2127
2128 fn has_named(&self, project: &str, name: &str) -> bool {
2130 name == "default"
2131 || self.workspace.projects[project]
2132 .named_inputs
2133 .contains_key(name)
2134 }
2135
2136 fn dependency_closure(&self, project: &str) -> BTreeSet<String> {
2138 let mut closure = BTreeSet::new();
2139 let mut pending = vec![project.to_owned()];
2140 while let Some(current) = pending.pop() {
2141 for edge in &self.snapshot.projects.dependencies[¤t] {
2142 if edge.target != project && closure.insert(edge.target.clone()) {
2143 pending.push(edge.target.clone());
2144 }
2145 }
2146 }
2147 closure
2148 }
2149
2150 fn dependencies_named(&mut self, project: &str, name: &str) -> Result<()> {
2152 for dependency in self.dependency_closure(project) {
2153 if self.expanded.insert((dependency.clone(), name.to_owned())) {
2154 self.named(&dependency, name)?;
2155 }
2156 }
2157 Ok(())
2158 }
2159
2160 fn json(&mut self, project: &str, object: &serde_json::Map<String, Value>) -> Result<()> {
2164 for key in object.keys() {
2165 if !matches!(key.as_str(), "json" | "fields" | "excludeFields") {
2166 bail!("unsupported json input field {key:?}");
2167 }
2168 }
2169 let declared = object["json"]
2170 .as_str()
2171 .context("json input must name a file")?;
2172 validate_fileset(declared)?;
2173 let path = paths::expand_glob(self.workspace, project, declared)?;
2174 let list = |name: &str| -> Result<Option<Vec<String>>> {
2175 object
2176 .get(name)
2177 .map(|value| {
2178 serde_json::from_value(value.clone())
2179 .with_context(|| format!("json input {name} must be an array of strings"))
2180 })
2181 .transpose()
2182 };
2183 let fields = JsonFields {
2184 fields: list("fields")?,
2185 exclude: list("excludeFields")?.unwrap_or_default(),
2186 };
2187 if crate::glob::names_a_path(&path) {
2188 let path = crate::glob::literal_path(&path);
2189 paths::validate_path(&path)?;
2190 return self.json_file(path, &fields);
2191 }
2192 let owner = if declared.starts_with("{projectRoot}") {
2195 Some(project)
2196 } else if declared.starts_with("{workspaceRoot}/") {
2197 None
2198 } else {
2199 return Ok(());
2200 };
2201 let matcher = self.snapshot.pattern(&path, false)?;
2202 let snapshot = self.snapshot;
2203 let matched: Vec<String> = self
2204 .candidates
2205 .iter()
2206 .filter(|file| matcher.is_match(file))
2207 .filter(|file| owner.is_none() || snapshot.owner(file) == owner)
2208 .cloned()
2209 .collect();
2210 for file in matched {
2211 self.json_file(file, &fields)?;
2212 }
2213 self.json_globs.push(JsonGlob {
2214 matcher,
2215 prefix: crate::ignored::partition(&path).0,
2216 owner: owner.map(str::to_owned),
2217 fields,
2218 });
2219 Ok(())
2220 }
2221
2222 fn json_file(&mut self, path: String, fields: &JsonFields) -> Result<()> {
2226 let text = match std::fs::read_to_string(self.workspace.root.join(&path)) {
2227 Ok(text) => Some(text),
2228 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
2229 Err(error) => return Err(error.into()),
2230 };
2231 let mut selections = self.json_fields.get(&path).cloned().unwrap_or_default();
2232 if !selections.contains(fields) {
2233 selections.push(fields.clone());
2234 selections.sort();
2236 }
2237 let mut values = Vec::new();
2238 for selection in &selections {
2239 values.push(match &text {
2240 Some(text) => match json_projection(&path, text, selection) {
2241 Some(value) => value,
2242 None => {
2246 self.selected.insert(path);
2247 return Ok(());
2248 }
2249 },
2250 None => Value::Null,
2251 });
2252 }
2253 let value = if values.len() == 1 {
2255 values.remove(0)
2256 } else {
2257 Value::Array(values)
2258 };
2259 self.values.insert(format!("json:{path}"), value);
2260 self.json_fields.insert(path, selections);
2261 Ok(())
2262 }
2263
2264 fn named(&mut self, project: &str, name: &str) -> Result<()> {
2265 let identity = (project.to_owned(), name.to_owned());
2266 if self.named_stack.contains(&identity) {
2267 bail!("named input cycle at {project}:{name}");
2268 }
2269 self.named_stack.push(identity);
2270 let inputs = self.workspace.projects[project]
2271 .named_inputs
2272 .get(name)
2273 .cloned()
2274 .or_else(|| (name == "default").then(|| vec![json!("{projectRoot}/**/*")]))
2275 .with_context(|| format!("unknown named input {name:?}"))?;
2276 for input in &inputs {
2277 self.input(project, input)?;
2278 }
2279 self.named_stack.pop();
2280 Ok(())
2281 }
2282}
2283
2284fn validate_fileset(fileset: &str) -> Result<()> {
2287 let body = fileset.strip_prefix('!').unwrap_or(fileset);
2288 if !body.starts_with("{projectRoot}") && !body.starts_with("{workspaceRoot}") {
2289 bail!(
2290 "{fileset:?} is an invalid fileset: filesets have to start with either {{workspaceRoot}} or {{projectRoot}}, and a named input must be defined"
2291 );
2292 }
2293 Ok(())
2294}
2295
2296fn declared_inputs(task: &Task) -> Vec<Value> {
2298 task.definition
2299 .inputs
2300 .clone()
2301 .unwrap_or_else(|| vec![json!("default"), json!("^default")])
2302}
2303
2304fn servers(graph: &TaskGraph, id: &str) -> Vec<(String, Vec<Value>)> {
2310 let mut servers = Vec::new();
2311 let mut visited = BTreeSet::from([id]);
2312 let mut pending = std::collections::VecDeque::from([id]);
2313 while let Some(current) = pending.pop_front() {
2314 let task = &graph.tasks[current];
2315 for dependency in &task.dependencies {
2316 let Some(server) = graph.tasks.get(dependency) else {
2317 continue;
2318 };
2319 if server.definition.continuous != Some(true)
2320 || !task.edge(dependency).inputs
2321 || !visited.insert(dependency)
2322 {
2323 continue;
2324 }
2325 servers.push((server.project.clone(), declared_inputs(server)));
2326 pending.push_back(dependency);
2327 }
2328 }
2329 servers
2330}
2331
2332pub fn resolve(
2333 snapshot: &Snapshot,
2334 workspace: &Workspace,
2335 task: &Task,
2336 prepared: Option<&PreparedTask>,
2337 cancelled: &AtomicBool,
2338) -> Result<Resolved> {
2339 resolve_candidates(
2340 snapshot,
2341 workspace,
2342 task,
2343 prepared,
2344 cancelled,
2345 &snapshot.files,
2346 &snapshot.source_ignore,
2347 )
2348}
2349
2350fn resolve_candidates(
2351 snapshot: &Snapshot,
2352 workspace: &Workspace,
2353 task: &Task,
2354 prepared: Option<&PreparedTask>,
2355 cancelled: &AtomicBool,
2356 candidates: &BTreeSet<String>,
2357 source_ignore: &SourceIgnore,
2358) -> Result<Resolved> {
2359 let mut resolver = Resolver::new(snapshot, workspace, prepared, cancelled, candidates);
2360 resolver.task_inputs(task)?;
2361 let mut upstream = BTreeSet::new();
2365 let mut pending: Vec<_> = task.dependencies.iter().cloned().collect();
2366 while let Some(id) = pending.pop() {
2367 if upstream.insert(id.clone())
2368 && let Some(dependencies) = snapshot.task_dependencies.get(&id)
2369 {
2370 pending.extend(dependencies.iter().cloned());
2371 }
2372 }
2373 let own_outputs = Outputs::new(workspace, task)
2374 .ok()
2375 .filter(Outputs::is_explicit);
2376 let mut producers = BTreeSet::new();
2377 for (producer, declared) in &snapshot.generated_tasks {
2378 if producer.id == task.id || !upstream.contains(&producer.id) {
2379 continue;
2380 }
2381 let overlaps = |a: &str, b: &str| {
2382 a.is_empty()
2383 || a == b
2384 || a.strip_prefix(b).is_some_and(|rest| rest.starts_with('/'))
2385 || b.strip_prefix(a).is_some_and(|rest| rest.starts_with('/'))
2386 };
2387 let relevant = resolver
2388 .scopes
2389 .values()
2390 .flat_map(|scope| &scope.patterns)
2391 .filter(|pattern| !pattern.excluded)
2392 .map(|pattern| &pattern.prefix)
2393 .chain(resolver.json_globs.iter().map(|glob| &glob.prefix))
2394 .any(|prefix| {
2395 declared.anchors().any(|anchor| {
2396 overlaps(prefix, anchor)
2397 && (!source_ignore.matches_directory(anchor)
2398 || !workspace.root.join(anchor).is_dir())
2399 })
2400 });
2401 if !relevant {
2402 continue;
2403 }
2404 producers.insert(producer.id.clone());
2405 let outputs = snapshot.dependency_outputs(workspace, producer)?;
2406 let mut json = Vec::new();
2407 for path in &outputs.files {
2408 if source_ignore.matches(path)
2409 || own_outputs.as_ref().is_some_and(|own| own.matches(path))
2410 {
2411 continue;
2412 }
2413 json.extend(
2414 resolver
2415 .json_globs
2416 .iter()
2417 .filter(|glob| {
2418 glob.matcher.is_match(path)
2419 && (glob.owner.is_none()
2420 || snapshot.owner(path) == glob.owner.as_deref())
2421 })
2422 .map(|glob| (path.clone(), glob.fields.clone())),
2423 );
2424 for selection in resolver.scopes.values_mut() {
2425 for pattern in &selection.patterns {
2426 if pattern.matcher.is_match(path)
2427 && (pattern.owner.is_none()
2428 || snapshot.owner(path) == pattern.owner.as_deref())
2429 {
2430 if pattern.excluded {
2431 selection.excluded.insert(path.clone());
2432 } else {
2433 selection.included.insert(path.clone());
2434 }
2435 }
2436 }
2437 }
2438 }
2439 for (path, fields) in json {
2440 resolver.json_file(path, &fields)?;
2441 }
2442 }
2443 for selection in resolver.scopes.values() {
2446 resolver.selected.extend(
2447 selection
2448 .included
2449 .difference(&selection.excluded)
2450 .filter(|path| {
2451 snapshot.extra_candidates.contains(*path)
2452 || own_outputs.as_ref().is_none_or(|own| !own.matches(path))
2453 })
2454 .cloned(),
2455 );
2456 }
2457 let mut ignored = BTreeSet::new();
2461 for ((project, _), globs) in &resolver.ignored {
2462 let group = Group::new(
2463 project,
2464 globs,
2465 workspace
2466 .projects
2467 .values()
2468 .map(|project| project.root.as_str()),
2469 )?;
2470 let found = group.expand(&workspace.root)?;
2471 ignored.extend(
2472 found
2473 .into_iter()
2474 .chain(
2475 snapshot
2476 .extra_candidates
2477 .iter()
2478 .filter(|path| group.matches(path))
2479 .cloned(),
2480 )
2481 .filter(|path| own_outputs.as_ref().is_none_or(|own| !own.matches(path))),
2483 );
2484 for (producer, declared) in &snapshot.generated_tasks {
2485 if producer.id != task.id
2486 && upstream.contains(&producer.id)
2487 && declared.declared()
2488 && reads_outputs(&group, declared)
2489 {
2490 producers.insert(producer.id.clone());
2491 }
2492 }
2493 }
2494 resolver.selected.extend(ignored.iter().cloned());
2495 let mut declared = Declared {
2496 files: resolver.selected,
2497 ignored,
2498 values: resolver.values,
2499 json_fields: resolver.json_fields,
2500 external: resolver.external,
2501 external_declared: resolver.external_declared,
2502 projects: resolver.projects,
2503 producers,
2504 selection: Selection {
2505 scopes: resolver
2506 .scopes
2507 .into_values()
2508 .map(|selection| selection.patterns)
2509 .collect(),
2510 },
2511 };
2512 let reads: Vec<bool> = declared
2515 .values
2516 .iter()
2517 .filter(|(name, _)| name.starts_with("dependentTasksOutputFiles:"))
2518 .map(|(_, transitive)| transitive == &Value::Bool(true))
2519 .collect();
2520 if !reads.is_empty() {
2521 let transitive = reads.contains(&true);
2522 for (producer, outputs) in &snapshot.generated_tasks {
2523 let read = if transitive {
2524 upstream.contains(&producer.id)
2525 } else {
2526 task.dependencies.contains(&producer.id)
2527 };
2528 if read && outputs.declared() {
2529 declared.producers.insert(producer.id.clone());
2530 }
2531 }
2532 }
2533 let mut inferred = infer(snapshot, workspace, task)?;
2534 if declared.external_declared
2538 && let Some(importers) = &mut inferred.lockfile
2539 {
2540 let root = workspace.projects[&task.project].root == ".";
2541 if !root {
2542 importers.remove(".");
2543 }
2544 }
2545 Ok(join(declared, inferred))
2546}
2547
2548fn reads_outputs(group: &Group, outputs: &Outputs) -> bool {
2554 let holds = |outer: &str, inner: &str| {
2555 outer.is_empty()
2556 || inner == outer
2557 || inner
2558 .strip_prefix(outer)
2559 .is_some_and(|rest| rest.starts_with('/'))
2560 };
2561 group.reads().any(|(root, extension)| {
2562 outputs.globs().iter().any(|output| {
2563 let (prefix, _) = crate::ignored::partition(output);
2564 (holds(root, &prefix) || holds(&prefix, root))
2565 && !group
2566 .skipped_below(root)
2567 .iter()
2568 .any(|project| holds(project, &prefix))
2569 && match (extension, crate::ignored::extension(output)) {
2570 (Some(read), Some(written)) => read == written,
2571 _ => true,
2572 }
2573 })
2574 })
2575}
2576
2577fn infer(snapshot: &Snapshot, workspace: &Workspace, task: &Task) -> Result<Inferred> {
2579 let readable = snapshot
2580 .installed(&workspace.root, &task.id)?
2581 .is_some_and(|installed| installed.lockfile.is_some());
2582 let exists = |path: &str| {
2583 workspace.root.join(path).is_file() || snapshot.extra_candidates.contains(path)
2584 };
2585 let mut inferred = Inferred {
2586 lockfile_readable: readable,
2587 ..Inferred::default()
2588 };
2589 inferred.tsconfig = crate::inputs::ROOT_TSCONFIG_FILES
2596 .into_iter()
2597 .find(|name| exists(name))
2598 .map(str::to_owned);
2599 for path in ["nx.json", qk_config::LOCAL_WORKSPACE] {
2603 if exists(path) {
2604 inferred.settings.insert(path.into());
2605 }
2606 }
2607 if let Some(extended) = &workspace.extended {
2608 inferred.settings.insert(extended.clone());
2609 }
2610 for path in [
2611 ".gitignore",
2612 ".nxignore",
2613 "package-lock.json",
2614 "yarn.lock",
2615 "bun.lock",
2616 ] {
2617 if exists(path) {
2618 inferred.files.insert(path.into());
2619 }
2620 }
2621 if exists("pnpm-workspace.yaml") {
2625 if readable {
2626 inferred.workspace_file = true;
2627 } else {
2628 inferred.files.insert("pnpm-workspace.yaml".into());
2629 }
2630 }
2631 if exists("pnpm-lock.yaml") && !readable {
2632 inferred.files.insert("pnpm-lock.yaml".into());
2633 }
2634 let mut packages = BTreeSet::from([task.project.clone()]);
2639 let mut pending = packages.clone();
2640 while let Some(project) = pending.pop_first() {
2641 for edge in &snapshot.projects.dependencies[&project] {
2642 if packages.insert(edge.target.clone()) {
2643 pending.insert(edge.target.clone());
2644 }
2645 }
2646 }
2647 for project in &packages {
2648 let own = *project == task.project;
2649 for name in ["project.json", qk_config::LOCAL_OVERRIDES, "package.json"] {
2650 if !own && name != "package.json" {
2651 continue;
2652 }
2653 let path = Path::new(&workspace.projects[project].root).join(name);
2654 let path = path.strip_prefix(".").unwrap_or(&path).to_owned();
2655 if !workspace.root.join(&path).is_file() {
2656 continue;
2657 }
2658 let path = path
2659 .to_str()
2660 .context("project path must be UTF-8")?
2661 .replace('\\', "/");
2662 if own && name != "package.json" {
2667 inferred.project.insert(path);
2668 continue;
2669 }
2670 if own {
2671 inferred.files.insert(path);
2672 continue;
2673 }
2674 let text = std::fs::read_to_string(workspace.root.join(&path))
2675 .with_context(|| format!("cannot read {path}"))?;
2676 match manifest_for_dependents(&text, readable) {
2677 Some(value) => {
2678 inferred.manifests.insert(path, value);
2679 }
2680 None => {
2681 inferred.files.insert(path);
2682 }
2683 }
2684 }
2685 }
2686 if readable {
2687 inferred.lockfile = Some(
2690 std::iter::once(".".to_owned())
2691 .chain(
2692 packages
2693 .iter()
2694 .map(|project| workspace.projects[project].root.clone()),
2695 )
2696 .collect(),
2697 );
2698 }
2699 Ok(inferred)
2700}
2701
2702fn join(declared: Declared, mut inferred: Inferred) -> Resolved {
2705 let mut covered = BTreeSet::new();
2706 inferred.files.retain(|path| {
2707 let keep = !declared.files.contains(path);
2708 if !keep {
2709 covered.insert(path.clone());
2710 }
2711 keep
2712 });
2713 inferred.manifests.retain(|path, _| {
2714 let keep = !declared.files.contains(path);
2715 if !keep {
2716 covered.insert(path.clone());
2717 }
2718 keep
2719 });
2720 if inferred.lockfile.is_some() && declared.files.contains("pnpm-lock.yaml") {
2721 inferred.lockfile = None;
2722 covered.insert("pnpm-lock.yaml".into());
2723 }
2724 if inferred.workspace_file && declared.files.contains("pnpm-workspace.yaml") {
2725 inferred.workspace_file = false;
2726 covered.insert("pnpm-workspace.yaml".into());
2727 }
2728 if let Some(tsconfig) = inferred
2729 .tsconfig
2730 .take_if(|path| declared.files.contains(path))
2731 {
2732 covered.insert(tsconfig);
2733 }
2734 inferred.covered = covered;
2735 Resolved { declared, inferred }
2736}
2737
2738pub fn inputs(
2741 snapshot: &Snapshot,
2742 workspace: &Workspace,
2743 task: &Task,
2744 prepared: &PreparedTask,
2745 dependencies: &BTreeMap<String, String>,
2746 cancelled: &AtomicBool,
2747) -> Result<Value> {
2748 let prefixes = snapshot.membership_prefixes(workspace, task)?;
2749 let candidates = snapshot.current_files_for(&workspace.root, Some(&prefixes), &task.id)?;
2750 let Resolved { declared, inferred } = resolve_candidates(
2751 snapshot,
2752 workspace,
2753 task,
2754 Some(prepared),
2755 cancelled,
2756 &candidates.files,
2757 &candidates.source_ignore,
2758 )?;
2759 let file_values = |paths: &BTreeSet<String>| {
2760 paths
2761 .iter()
2762 .map(|path| {
2763 let value = snapshot
2764 .file_value(&workspace.root, path)
2765 .with_context(|| format!("cannot read input {path}"))?;
2766 Ok((path.clone(), value))
2767 })
2768 .collect::<Result<BTreeMap<_, _>>>()
2769 };
2770 let mut files = file_values(&(&declared.files - &declared.ignored))?;
2771 for path in &declared.ignored {
2772 if let Some(value) = snapshot
2773 .ignored_value(&workspace.root, path)
2774 .with_context(|| format!("cannot read input {path}"))?
2775 {
2776 files.insert(path.clone(), value);
2777 }
2778 }
2779 let mut values = declared.values;
2780 let mut configuration = json!({
2781 "files": file_values(&inferred.files)?,
2782 "settings": workspace.config.keyed_settings(),
2783 "project": task_configuration(workspace, task),
2784 });
2785 let installed = snapshot.installed(&workspace.root, &task.id)?;
2786 if let (Some(importers), Some(installed)) = (&inferred.lockfile, &installed)
2787 && let Some(lockfile) = &installed.lockfile
2788 {
2789 let importers: BTreeMap<_, _> = importers
2790 .iter()
2791 .filter(|importer| lockfile.has_importer(importer))
2792 .map(|importer| {
2793 let digest = installed.digest(&format!("importer:{importer}"), || {
2794 lockfile.installed(importer).unwrap_or_default()
2795 });
2796 (importer.clone(), digest)
2797 })
2798 .collect();
2799 configuration["lockfile"] = json!({
2800 "global": installed.digest("global", || BTreeSet::from([lockfile.global().to_string()])),
2801 "importers": importers,
2802 });
2803 if !declared.external.is_empty() {
2805 let external: BTreeMap<_, _> = declared
2806 .external
2807 .iter()
2808 .map(|name| {
2809 let digest =
2810 installed.digest(&format!("package:{name}"), || lockfile.package(name));
2811 if digest == blake3::hash(b"").to_hex().as_str() {
2813 bail!(
2814 "the externalDependency {name:?} of {} could not be found",
2815 task.id
2816 );
2817 }
2818 Ok((name.clone(), digest))
2819 })
2820 .collect::<Result<_>>()?;
2821 values.insert("externalDependencies".into(), json!(external));
2822 }
2823 }
2824 if inferred.workspace_file {
2825 configuration["workspaceFile"] = match workspace_without_resolution(&workspace.root) {
2826 Some(value) => value,
2827 None => snapshot.file_value(&workspace.root, "pnpm-workspace.yaml")?,
2828 };
2829 }
2830 if !inferred.manifests.is_empty() {
2831 configuration["manifests"] = json!(inferred.manifests);
2832 }
2833 if let Some(path) = &inferred.tsconfig {
2834 let parsed = std::fs::read_to_string(workspace.root.join(path))
2835 .ok()
2836 .and_then(|text| TsConfig::parse(&text));
2837 configuration["tsconfig"] = match parsed {
2838 Some(tsconfig) if crate::inputs::selectively_hash_tsconfig(workspace) => json!({
2839 "rest": tsconfig.rest,
2840 "paths": tsconfig.paths_of(&declared.projects, |target| snapshot.owner(target)),
2841 }),
2842 Some(tsconfig) => json!({ "rest": tsconfig.rest }),
2843 None => snapshot.file_value(&workspace.root, path)?,
2844 };
2845 }
2846 let definition = keyed_definition(task);
2847 let hash = json!({
2848 "schema":"qk-local-v4", "qk":env!("CARGO_PKG_VERSION"),
2849 "platform":[std::env::consts::OS, std::env::consts::ARCH], "workspace":snapshot.workspace_prefix,
2850 "id":task.id, "args":task.args, "definition":definition, "packageManager":workspace.package_manager,
2851 "files":files, "values":values, "inferred":configuration, "dependencies":dependencies,
2852 });
2853 Ok(hash)
2854}
2855
2856pub fn keyed_definition(task: &Task) -> qk_config::Target {
2860 let mut definition = task.definition.clone();
2861 definition.extra.remove("qk:warm");
2862 definition.extra.remove("qk:threads");
2863 definition.extra.remove("metadata");
2864 definition.extra.remove("qk:ultracache");
2866 definition
2867}
2868
2869pub fn task_configuration(workspace: &Workspace, task: &Task) -> Value {
2875 let project = &workspace.projects[&task.project];
2876 let mut value = json!({ "settings": project.keyed_settings() });
2877 if let Ok((outputs, _)) = paths::resolved_outputs(workspace, task) {
2878 value["outputs"] = json!(outputs);
2879 }
2880 if task.args.iter().any(|arg| arg.contains("{project")) {
2881 value["configuration"] = serde_json::to_value(project).unwrap_or_default();
2882 }
2883 value
2884}
2885
2886pub fn key(inputs: &Value) -> Result<String> {
2890 Ok(blake3::hash(&serde_json::to_vec(&sorted(inputs))?)
2891 .to_hex()
2892 .to_string())
2893}
2894
2895fn sorted(value: &Value) -> Value {
2897 match value {
2898 Value::Object(object) => {
2899 let mut fields: Vec<_> = object.iter().collect();
2900 fields.sort_unstable_by_key(|(name, _)| *name);
2901 Value::Object(
2902 fields
2903 .into_iter()
2904 .map(|(name, value)| (name.clone(), sorted(value)))
2905 .collect(),
2906 )
2907 }
2908 Value::Array(items) => Value::Array(items.iter().map(sorted).collect()),
2909 other => other.clone(),
2910 }
2911}
2912
2913pub fn fingerprint(
2914 snapshot: &Snapshot,
2915 workspace: &Workspace,
2916 task: &Task,
2917 prepared: &PreparedTask,
2918 dependencies: &BTreeMap<String, String>,
2919 cancelled: &AtomicBool,
2920) -> Result<String> {
2921 key(&inputs(
2922 snapshot,
2923 workspace,
2924 task,
2925 prepared,
2926 dependencies,
2927 cancelled,
2928 )?)
2929}
2930
2931#[allow(clippy::too_many_arguments)]
2936pub(crate) fn weak_key(
2937 snapshot: &Snapshot,
2938 workspace: &Workspace,
2939 graph: &TaskGraph,
2940 task: &Task,
2941 prepared: &PreparedTask,
2942 inputs: &Value,
2943 ignore_reads: &[String],
2944 cancelled: &AtomicBool,
2945) -> Result<String> {
2946 let mut weak = inputs.clone();
2947 let object = weak
2948 .as_object_mut()
2949 .context("a key's inputs are an object")?;
2950 object.remove("files");
2951 object.remove("dependencies");
2952 if let Some(Value::Object(values)) = object.get_mut("values") {
2953 values.retain(|name, _| {
2954 !name.starts_with("json:") && !name.starts_with("dependentTasksOutputFiles:")
2955 });
2956 }
2957 let mut always = task.clone();
2958 always.definition.inputs = Some(
2959 declared_inputs(task)
2960 .into_iter()
2961 .filter(|input| input.get("always") == Some(&Value::Bool(true)))
2962 .collect(),
2963 );
2964 let dependencies = dependency_keys(snapshot, workspace, graph, &always, cancelled)?;
2965 let always = self::inputs(
2966 snapshot,
2967 workspace,
2968 &always,
2969 prepared,
2970 &dependencies,
2971 cancelled,
2972 )?;
2973 object.insert(
2974 "always".into(),
2975 json!({"files": always["files"], "values": always["values"], "dependencies": always["dependencies"]}),
2976 );
2977 object.insert("schema".into(), json!("qk-weak-v1"));
2978 object.insert("ignoreReads".into(), json!(ignore_reads));
2979 key(&weak)
2980}
2981
2982pub(crate) fn dependency_keys(
2987 snapshot: &Snapshot,
2988 workspace: &Workspace,
2989 graph: &TaskGraph,
2990 task: &Task,
2991 cancelled: &AtomicBool,
2992) -> Result<BTreeMap<String, String>> {
2993 dependency_keys_impl(snapshot, workspace, graph, task, cancelled, false)
2994}
2995
2996pub(crate) fn recheck_dependency_keys(
2998 snapshot: &Snapshot,
2999 workspace: &Workspace,
3000 graph: &TaskGraph,
3001 task: &Task,
3002 cancelled: &AtomicBool,
3003) -> Result<BTreeMap<String, String>> {
3004 dependency_keys_impl(snapshot, workspace, graph, task, cancelled, true)
3005}
3006
3007fn dependency_keys_impl(
3008 snapshot: &Snapshot,
3009 workspace: &Workspace,
3010 graph: &TaskGraph,
3011 task: &Task,
3012 cancelled: &AtomicBool,
3013 fresh: bool,
3014) -> Result<BTreeMap<String, String>> {
3015 let resolved = resolve(snapshot, workspace, task, None, cancelled)?;
3016 let selections: Vec<_> = resolved.dependency_output_inputs().collect();
3017 let mut keys = BTreeMap::new();
3018 let mut fresh_outputs = HashMap::new();
3019 for (pattern, transitive) in selections {
3020 let matcher = snapshot.pattern(pattern, false)?;
3021 let mut selected = task.dependencies.clone();
3022 if transitive {
3023 let mut pending = selected.clone();
3024 while let Some(id) = pending.pop_first() {
3025 for dependency in &graph.tasks[&id].dependencies {
3026 if selected.insert(dependency.clone()) {
3027 pending.insert(dependency.clone());
3028 }
3029 }
3030 }
3031 }
3032 for id in selected {
3033 if !Outputs::new(workspace, &graph.tasks[&id])?.declared() {
3035 continue;
3036 }
3037 let outputs = if fresh {
3038 if !fresh_outputs.contains_key(&id) {
3039 fresh_outputs.insert(
3040 id.clone(),
3041 Arc::new(snapshot.fresh_dependency_outputs(workspace, &graph.tasks[&id])?),
3042 );
3043 }
3044 fresh_outputs[&id].clone()
3045 } else {
3046 snapshot.dependency_outputs(workspace, &graph.tasks[&id])?
3047 };
3048 let mut files = BTreeMap::new();
3049 for path in outputs.files.iter().filter(|path| matcher.is_match(path)) {
3050 let known = outputs.values.lock().unwrap().get(path).cloned();
3051 let value = match known {
3052 Some(value) => value,
3053 None => {
3054 let value = snapshot.file_value(&workspace.root, path)?;
3055 if value.get("link").is_none() {
3057 outputs
3058 .values
3059 .lock()
3060 .unwrap()
3061 .insert(path.clone(), value.clone());
3062 }
3063 value
3064 }
3065 };
3066 files.insert(path.clone(), value);
3067 }
3068 keys.insert(format!("{id}:{pattern}"), key(&json!(files))?);
3069 }
3070 }
3071 Ok(keys)
3072}
3073
3074#[derive(Clone, Debug, PartialEq, Eq)]
3077pub struct DependencyGlob {
3078 pub task: String,
3079 pub pattern: String,
3080 pub selects: Selects,
3081}
3082
3083#[derive(Clone, Debug, PartialEq, Eq)]
3084pub enum Selects {
3085 NoDeclaredOutputs,
3087 NotBuilt,
3089 NoneOf(usize),
3091 Some,
3093}
3094
3095pub fn dependency_globs(workspace: &Workspace, graph: &TaskGraph) -> Result<Vec<DependencyGlob>> {
3098 let snapshot = Snapshot::new(workspace, graph, &paths::cache_location(workspace))?;
3099 let cancelled = AtomicBool::new(false);
3100 let mut globs = Vec::new();
3101 for task in graph.tasks.values() {
3102 let Ok(resolved) = resolve(&snapshot, workspace, task, None, &cancelled) else {
3104 continue;
3105 };
3106 for (pattern, transitive) in resolved.dependency_output_inputs() {
3107 let mut selected = task.dependencies.clone();
3108 if transitive {
3109 let mut pending = selected.clone();
3110 while let Some(id) = pending.pop_first() {
3111 for dependency in &graph.tasks[&id].dependencies {
3112 if selected.insert(dependency.clone()) {
3113 pending.insert(dependency.clone());
3114 }
3115 }
3116 }
3117 }
3118 let matcher = snapshot.pattern(pattern, false)?;
3119 let (mut declared, mut files, mut matched, mut unbuilt) = (false, 0, false, false);
3121 for id in &selected {
3122 let dependency = &graph.tasks[id];
3123 if !Outputs::new(workspace, dependency)?.declared() {
3124 continue;
3125 }
3126 declared = true;
3127 let outputs = snapshot.dependency_outputs(workspace, dependency)?;
3128 unbuilt |= outputs.files.is_empty();
3129 files += outputs.files.len();
3130 matched |= outputs.files.iter().any(|path| matcher.is_match(path));
3131 }
3132 globs.push(DependencyGlob {
3133 task: task.id.clone(),
3134 pattern: pattern.to_owned(),
3135 selects: match (declared, matched, unbuilt) {
3136 (false, ..) => Selects::NoDeclaredOutputs,
3137 (_, true, _) => Selects::Some,
3138 (_, _, true) => Selects::NotBuilt,
3139 _ => Selects::NoneOf(files),
3140 },
3141 });
3142 }
3143 }
3144 Ok(globs)
3145}
3146
3147#[cfg(test)]
3148mod tests {
3149
3150 fn whole(resolved: &Resolved) -> BTreeSet<String> {
3152 resolved.whole_files().map(str::to_owned).collect()
3153 }
3154 use super::*;
3155 use qk_taskgraph::Request;
3156
3157 fn fixture(outputs: Option<Value>) -> (tempfile::TempDir, Workspace, TaskGraph) {
3158 let root = tempfile::tempdir().unwrap();
3159 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
3160 let mut post = json!({"command": "true"});
3161 if let Some(outputs) = outputs {
3162 post["outputs"] = outputs;
3163 }
3164 std::fs::write(root.path().join("project.json"), json!({"name":"app", "targets": {
3165 "gen": {"command":"true", "outputs":["{projectRoot}/generated"]},
3166 "post": post,
3167 "build": {"command":"true", "dependsOn":["gen", "post"], "inputs":[{"dependentTasksOutputFiles":"**/*"}]}
3168 }}).to_string()).unwrap();
3169 std::fs::create_dir(root.path().join("generated")).unwrap();
3170 std::fs::write(root.path().join("generated/value"), "one").unwrap();
3171 let workspace = Workspace::load(root.path()).unwrap();
3172 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3173 (root, workspace, graph)
3174 }
3175
3176 #[test]
3180 fn keys_ignore_the_order_object_fields_were_written_in() {
3181 let written = json!({"options": {"cwd": "app", "command": "x"}, "args": ["a", "b"]});
3182 let reordered = json!({"args": ["a", "b"], "options": {"command": "x", "cwd": "app"}});
3183 assert_eq!(key(&written).unwrap(), key(&reordered).unwrap());
3184 let swapped = json!({"options": {"cwd": "app", "command": "x"}, "args": ["b", "a"]});
3185 assert_ne!(key(&written).unwrap(), key(&swapped).unwrap());
3186 }
3187 #[test]
3188 fn json_globs_select_dependency_outputs() {
3189 let root = tempfile::tempdir().unwrap();
3190 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
3191 for (path, content) in [
3192 (
3193 "apps/app/project.json",
3194 json!({"name": "app", "implicitDependencies": ["lib"], "targets": {
3195 "build": {"command": "true", "dependsOn": ["^build"], "inputs": [
3196 {"json": "{workspaceRoot}/libs/lib/out/*.json", "fields": ["version"]}
3197 ]}
3198 }}),
3199 ),
3200 (
3201 "libs/lib/project.json",
3202 json!({"name": "lib", "targets": {
3203 "build": {"command": "true", "outputs": ["{projectRoot}/out"]}
3204 }}),
3205 ),
3206 (
3207 "libs/lib/out/meta.json",
3208 json!({"version": 2, "built": "now"}),
3209 ),
3210 ] {
3211 let path = root.path().join(path);
3212 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
3213 std::fs::write(path, content.to_string()).unwrap();
3214 }
3215 let workspace = Workspace::load(root.path()).unwrap();
3216 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3217 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3218 let resolved = resolve(
3219 &snapshot,
3220 &workspace,
3221 &graph.tasks["app:build"],
3222 None,
3223 &AtomicBool::new(false),
3224 )
3225 .unwrap();
3226 assert_eq!(
3227 resolved.declared.values.get("json:libs/lib/out/meta.json"),
3228 Some(&json!({"version": 2}))
3229 );
3230 assert_eq!(
3231 resolved.declared.producers,
3232 BTreeSet::from(["lib:build".to_owned()])
3233 );
3234 }
3235
3236 #[test]
3240 fn declared_inputs_cover_what_they_would_select_and_not_what_they_leave_out() {
3241 let root = tempfile::tempdir().unwrap();
3242 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
3243 std::fs::write(
3244 root.path().join("project.json"),
3245 json!({"name": "app", "targets": {
3246 "build": {"command": "true", "inputs": [
3247 "{projectRoot}/src/**/*", "!{projectRoot}/src/skip/**/*"
3248 ]},
3249 "gen": {"command": "true", "outputs": ["{projectRoot}/src/gen"]}
3250 }})
3251 .to_string(),
3252 )
3253 .unwrap();
3254 std::fs::write(root.path().join(".gitignore"), "src/secret.ts\n").unwrap();
3255 for path in [
3256 "src/a.ts",
3257 "src/lib/b.ts",
3258 "src/skip/c.ts",
3259 "src/gen/d.ts",
3260 "README.md",
3261 ] {
3262 let path = root.path().join(path);
3263 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
3264 std::fs::write(path, "x").unwrap();
3265 }
3266 let workspace = Workspace::load(root.path()).unwrap();
3267 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3268 let task = &graph.tasks["app:build"];
3269 let prepared = qk_executor::prepare(&workspace, task, &BTreeMap::new()).unwrap();
3270 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3271 let coverage = coverage(
3272 &snapshot,
3273 &workspace,
3274 &graph,
3275 task,
3276 &prepared,
3277 &AtomicBool::new(false),
3278 )
3279 .unwrap();
3280 let set = |part: &str, path: &str| {
3281 let mut set = crate::recorded::PathSet::default();
3282 match part {
3283 "read" => &mut set.reads,
3284 "present" => &mut set.present,
3285 "absent" => &mut set.absent,
3286 _ => &mut set.listings,
3287 }
3288 .insert(path.to_owned());
3289 set
3290 };
3291 for (part, path, covered) in [
3292 ("read", "src/a.ts", true),
3293 ("read", "README.md", false),
3294 ("read", "src/skip/c.ts", false),
3295 ("present", "src/lib", true),
3296 ("absent", "src/new.ts", true),
3297 ("absent", "src/skip/new.ts", false),
3299 ("absent", "src/secret.ts", false),
3300 ("absent", ".env.local", false),
3301 ("absent", "src/gen/e.ts", false),
3303 ("listing", "src/gen", false),
3304 ("listing", "src/lib", true),
3305 ("listing", "src", false),
3307 ("listing", ".", false),
3308 ] {
3309 let generated = |path: &str| path == "src/gen" || path.starts_with("src/gen/");
3310 assert_eq!(
3311 coverage.covers(&snapshot, root.path(), &set(part, path), &generated),
3312 covered,
3313 "{part} {path}"
3314 );
3315 }
3316 }
3317
3318 #[test]
3319 fn execution_fingerprints_refresh_sources_and_preserve_artifact_exclusions() {
3320 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/dist"])));
3321 let definition = workspace
3322 .projects
3323 .get_mut("app")
3324 .unwrap()
3325 .targets
3326 .get_mut("post")
3327 .unwrap();
3328 definition
3329 .extra
3330 .insert("qk:warm".into(), json!({"paths": ["scratch"]}));
3331 std::fs::write(root.path().join(".gitignore"), "ignored/\n").unwrap();
3332 std::fs::create_dir(root.path().join("src")).unwrap();
3333 std::fs::write(root.path().join("src/original"), "one").unwrap();
3334 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3335 let task = &graph.tasks["app:post"];
3336 let prepared = qk_executor::prepare(&workspace, task, &BTreeMap::new()).unwrap();
3337 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3338 let inputs = || {
3339 super::inputs(
3340 &snapshot,
3341 &workspace,
3342 task,
3343 &prepared,
3344 &BTreeMap::new(),
3345 &AtomicBool::new(false),
3346 )
3347 .unwrap()
3348 };
3349 let before = inputs();
3350 for directory in ["generated", "dist", "scratch", "cache", "ignored"] {
3352 std::fs::create_dir_all(root.path().join(directory)).unwrap();
3353 std::fs::write(root.path().join(directory).join("new"), "artifact").unwrap();
3354 }
3355 assert_eq!(before, inputs());
3356 std::fs::write(root.path().join("src/added"), "two").unwrap();
3359 let after = inputs();
3360 assert!(after["files"].get("src/added").is_some());
3361 assert_ne!(key(&before).unwrap(), key(&after).unwrap());
3362 std::fs::remove_file(root.path().join("src/added")).unwrap();
3363 assert_eq!(before, inputs());
3364 std::fs::remove_file(root.path().join("src/original")).unwrap();
3365 assert!(inputs()["files"].get("src/original").is_none());
3366 }
3367
3368 #[test]
3369 fn membership_reuses_settled_candidates_and_detects_directory_changes() {
3370 let mut fixtures = Vec::new();
3371 for git in [false, true] {
3372 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3373 for directory in ["src/empty", "dist", "cache", "ignored"] {
3374 std::fs::create_dir_all(root.path().join(directory)).unwrap();
3375 }
3376 std::fs::write(root.path().join(".gitignore"), "ignored/\n").unwrap();
3377 std::fs::write(root.path().join("src/original"), "one").unwrap();
3378 if git {
3379 assert!(
3380 paths::git(root.path(), &["init", "-q"])
3381 .unwrap()
3382 .status
3383 .success()
3384 );
3385 assert!(
3386 paths::git(root.path(), &["add", "."])
3387 .unwrap()
3388 .status
3389 .success()
3390 );
3391 }
3392 let snapshot =
3393 Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3394 snapshot.current_files(root.path()).unwrap();
3395 fixtures.push((root, snapshot));
3396 }
3397 std::thread::sleep(Duration::from_millis(2100));
3400 for (root, snapshot) in fixtures {
3401 let files = || snapshot.current_files(root.path()).unwrap();
3402 let before = files();
3403 assert!(Arc::ptr_eq(&before, &files()));
3404 for directory in ["generated", "dist", "cache"] {
3405 std::fs::write(root.path().join(directory).join("new"), "artifact").unwrap();
3406 }
3407 assert!(Arc::ptr_eq(&before, &files()));
3408 std::fs::write(root.path().join("src/empty/added"), "two").unwrap();
3409 assert!(files().contains("src/empty/added"));
3410 std::fs::create_dir_all(root.path().join("src/new/nested")).unwrap();
3411 std::fs::write(root.path().join("src/new/nested/added"), "three").unwrap();
3412 assert!(files().contains("src/new/nested/added"));
3413 std::fs::rename(
3414 root.path().join("src/original"),
3415 root.path().join("src/renamed"),
3416 )
3417 .unwrap();
3418 let renamed = files();
3419 assert!(!renamed.contains("src/original"));
3420 assert!(renamed.contains("src/renamed"));
3421 std::fs::remove_file(root.path().join("src/renamed")).unwrap();
3422 assert!(!files().contains("src/renamed"));
3423 std::fs::write(root.path().join(".gitignore"), "ignored/\nsrc/empty/\n").unwrap();
3426 assert!(!files().contains("src/empty/added"));
3427 std::fs::write(root.path().join(".gitignore"), "ignored/\n").unwrap();
3428 assert!(files().contains("src/empty/added"));
3429 }
3430 }
3431
3432 #[test]
3433 fn identical_ignore_rewrites_reuse_candidates_and_changed_rules_refresh() {
3434 for name in [".gitignore", ".nxignore"] {
3435 let (root, workspace, graph) = fixture(None);
3436 let policy = root.path().join(name);
3437 std::fs::write(root.path().join("alpha"), "a").unwrap();
3438 std::fs::write(root.path().join("bravo"), "b").unwrap();
3439 std::fs::write(&policy, "alpha\n").unwrap();
3440 let snapshot =
3441 Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3442 let original = snapshot.current_files(root.path()).unwrap();
3443 assert!(!original.contains("alpha"));
3444 assert!(original.contains("bravo"));
3445 for _ in 0..3 {
3446 std::fs::write(&policy, "alpha\n").unwrap();
3447 assert!(Arc::ptr_eq(
3448 &original,
3449 &snapshot.current_files(root.path()).unwrap()
3450 ));
3451 }
3452 std::fs::write(&policy, "bravo\n").unwrap();
3455 {
3456 let mut cached = snapshot.membership.lock().unwrap();
3457 Arc::make_mut(cached.as_mut().unwrap())
3458 .stamps
3459 .insert(policy.clone(), membership_stamp(&policy).unwrap());
3460 }
3461 let changed = snapshot.current_files(root.path()).unwrap();
3462 assert!(!Arc::ptr_eq(&original, &changed));
3463 assert!(changed.contains("alpha"));
3464 assert!(!changed.contains("bravo"));
3465 std::fs::remove_file(&policy).unwrap();
3466 assert!(
3467 snapshot
3468 .current_files(root.path())
3469 .unwrap()
3470 .contains("bravo")
3471 );
3472 }
3473 }
3474
3475 #[test]
3476 fn ignore_contents_changed_after_inventory_cannot_bless_a_listing() {
3477 let (root, workspace, graph) = fixture(None);
3478 let policy = root.path().join(".nxignore");
3479 std::fs::write(&policy, "alpha\n").unwrap();
3480 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3481 snapshot.current_files(root.path()).unwrap();
3482 let membership = snapshot.membership.lock().unwrap().clone().unwrap();
3483 std::fs::write(&policy, "bravo\n").unwrap();
3484 assert!(!membership.state(None).unwrap().0);
3485 }
3486
3487 #[test]
3488 fn a_directory_missed_by_the_inventory_cannot_enable_reuse() {
3489 let (root, workspace, graph) = fixture(None);
3490 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3491 snapshot.current_files(root.path()).unwrap();
3492 std::fs::create_dir(root.path().join("new-empty")).unwrap();
3493 let mut cached = snapshot.membership.lock().unwrap();
3494 let membership = Arc::make_mut(cached.as_mut().unwrap());
3495 membership
3498 .directory_entries
3499 .get_mut(root.path())
3500 .unwrap()
3501 .insert((OsString::from("new-empty"), 1));
3502 assert!(membership.entries(root.path()).unwrap().is_none());
3503 assert!(!membership.state(None).unwrap().0);
3504 }
3505
3506 #[test]
3507 fn a_policy_missed_by_discovery_cannot_enable_reuse() {
3508 let (root, workspace, graph) = fixture(None);
3509 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3510 snapshot.current_files(root.path()).unwrap();
3511 std::fs::write(root.path().join(".nxignore"), "hidden\n").unwrap();
3512 let mut cached = snapshot.membership.lock().unwrap();
3513 let membership = Arc::make_mut(cached.as_mut().unwrap());
3514 membership
3517 .directory_entries
3518 .get_mut(root.path())
3519 .unwrap()
3520 .insert((OsString::from(".nxignore"), 0));
3521 assert!(membership.entries(root.path()).unwrap().is_none());
3522 assert!(!membership.state(None).unwrap().0);
3523 }
3524
3525 #[test]
3526 fn output_publication_reuses_candidates_but_source_names_still_refresh() {
3527 for partial in [false, true] {
3528 let outputs = if partial {
3529 json!(["{projectRoot}/dist", "!{projectRoot}/dist/keep"])
3530 } else {
3531 json!(["{projectRoot}/dist"])
3532 };
3533 let (root, workspace, graph) = fixture(Some(outputs));
3534 std::fs::create_dir_all(root.path().join("src")).unwrap();
3535 std::fs::write(root.path().join("src/original"), "one").unwrap();
3536 if partial {
3537 std::fs::create_dir_all(root.path().join("dist/keep")).unwrap();
3538 }
3539 let snapshot =
3540 Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3541 snapshot.current_files(root.path()).unwrap();
3542 std::thread::sleep(Duration::from_millis(2100));
3545 let before = snapshot.current_files(root.path()).unwrap();
3546 std::fs::create_dir_all(root.path().join("dist")).unwrap();
3547 std::fs::write(root.path().join("dist/restored"), "artifact").unwrap();
3548 let restored = snapshot.current_files(root.path()).unwrap();
3549 assert!(Arc::ptr_eq(&before, &restored));
3550 std::fs::remove_file(root.path().join("dist/restored")).unwrap();
3551 assert!(Arc::ptr_eq(
3552 &before,
3553 &snapshot.current_files(root.path()).unwrap()
3554 ));
3555 let source = if partial { "dist/keep/new" } else { "src/new" };
3556 std::fs::write(root.path().join(source), "source").unwrap();
3557 let after = snapshot.current_files(root.path()).unwrap();
3558 assert!(after.contains(source));
3559 assert!(!Arc::ptr_eq(&before, &after));
3560 std::fs::remove_file(root.path().join(source)).unwrap();
3561 assert!(
3562 !snapshot
3563 .current_files(root.path())
3564 .unwrap()
3565 .contains(source)
3566 );
3567 }
3568 }
3569
3570 #[test]
3571 fn concurrent_fingerprints_refresh_without_replacing_newer_membership() {
3572 let (root, workspace, graph) = fixture(None);
3573 std::fs::create_dir_all(root.path().join("src")).unwrap();
3574 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3575 let original = snapshot.current_files(root.path()).unwrap();
3576 let barrier = std::sync::Barrier::new(9);
3577 std::thread::scope(|scope| {
3578 let workers: Vec<_> = (0..8)
3579 .map(|_| {
3580 scope.spawn(|| {
3581 let mut rounds = Vec::new();
3582 for _ in 0..6 {
3583 barrier.wait();
3584 rounds.push(snapshot.current_files(root.path()));
3585 barrier.wait();
3586 }
3587 rounds
3588 })
3589 })
3590 .collect();
3591 for round in 0..6 {
3592 std::fs::write(root.path().join(format!("src/new{round}")), "source").unwrap();
3593 barrier.wait();
3594 barrier.wait();
3595 }
3596 for worker in workers {
3599 for (round, files) in worker.join().unwrap().into_iter().enumerate() {
3600 let files = files.unwrap();
3601 for added in 0..=round {
3602 assert!(files.contains(&format!("src/new{added}")));
3603 }
3604 }
3605 }
3606 });
3607 assert!(!original.contains("src/new0"));
3608 assert!(
3609 snapshot
3610 .current_files(root.path())
3611 .unwrap()
3612 .contains("src/new5")
3613 );
3614 }
3615
3616 #[test]
3617 fn retained_candidates_keep_their_dependency_output_ignore_policy() {
3618 let (root, mut workspace, _) = fixture(None);
3619 workspace
3620 .projects
3621 .get_mut("app")
3622 .unwrap()
3623 .targets
3624 .get_mut("build")
3625 .unwrap()
3626 .inputs = Some(vec![json!("{workspaceRoot}/**/*.txt")]);
3627 std::fs::create_dir(root.path().join("src")).unwrap();
3628 for name in ["alpha", "beta", "gamma"] {
3629 for directory in ["src", "generated"] {
3630 std::fs::write(root.path().join(format!("{directory}/{name}.txt")), name).unwrap();
3631 }
3632 }
3633 let policy = |keep: &str| {
3634 let ignored: String = ["alpha", "beta", "gamma"]
3635 .into_iter()
3636 .filter(|name| *name != keep)
3637 .flat_map(|name| {
3638 [
3639 format!("src/{name}.txt\n"),
3640 format!("generated/{name}.txt\n"),
3641 ]
3642 })
3643 .collect();
3644 std::fs::write(root.path().join(".nxignore"), ignored).unwrap();
3645 };
3646 policy("alpha");
3647 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3648 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
3649 let first = snapshot
3650 .current_files_for(root.path(), None, "test")
3651 .unwrap();
3652 policy("beta");
3653 let second = snapshot
3654 .current_files_for(root.path(), None, "test")
3655 .unwrap();
3656 policy("gamma");
3657 let third = snapshot
3658 .current_files_for(root.path(), None, "test")
3659 .unwrap();
3660 let cancelled = AtomicBool::new(false);
3663 let expected = |name: &str| {
3664 BTreeSet::from([
3665 ".nxignore".into(),
3666 format!("src/{name}.txt"),
3667 format!("generated/{name}.txt"),
3668 ])
3669 };
3670 for (candidates, name) in [(first, "alpha"), (second, "beta"), (third, "gamma")] {
3671 let resolved = resolve_candidates(
3672 &snapshot,
3673 &workspace,
3674 &graph.tasks["app:build"],
3675 None,
3676 &cancelled,
3677 &candidates.files,
3678 &candidates.source_ignore,
3679 )
3680 .unwrap();
3681 assert_eq!(whole(&resolved), expected(name));
3682 }
3683 let inspected = resolve(
3685 &snapshot,
3686 &workspace,
3687 &graph.tasks["app:build"],
3688 None,
3689 &cancelled,
3690 )
3691 .unwrap();
3692 assert_eq!(whole(&inspected), expected("alpha"));
3693 }
3694
3695 #[test]
3696 fn scoped_membership_defers_unrelated_changes_until_a_matching_task_asks() {
3697 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/dist"])));
3698 let targets = &mut workspace.projects.get_mut("app").unwrap().targets;
3699 targets.get_mut("post").unwrap().inputs =
3700 Some(vec![json!("{workspaceRoot}/left/**/*.txt")]);
3701 targets.get_mut("gen").unwrap().inputs =
3702 Some(vec![json!("{workspaceRoot}/right/**/*.txt")]);
3703 targets.get_mut("build").unwrap().inputs = Some(vec![json!("{workspaceRoot}/**/*.txt")]);
3704 for directory in ["left/empty", "right/empty"] {
3705 std::fs::create_dir_all(root.path().join(directory)).unwrap();
3706 }
3707 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3708 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3709 snapshot.current_files(&workspace.root).unwrap();
3710 std::thread::sleep(Duration::from_millis(2100));
3711 let left = snapshot
3712 .membership_prefixes(&workspace, &graph.tasks["app:post"])
3713 .unwrap();
3714 let right = snapshot
3715 .membership_prefixes(&workspace, &graph.tasks["app:gen"])
3716 .unwrap();
3717 let all = snapshot
3718 .membership_prefixes(&workspace, &graph.tasks["app:build"])
3719 .unwrap();
3720 let before = snapshot
3721 .current_files_for(&workspace.root, Some(&left), "test")
3722 .unwrap();
3723 std::fs::write(root.path().join("right/empty/new.txt"), "right").unwrap();
3724 let unrelated = snapshot
3725 .current_files_for(&workspace.root, Some(&left), "test")
3726 .unwrap();
3727 assert!(Arc::ptr_eq(&before.files, &unrelated.files));
3728 assert!(!unrelated.files.contains("right/empty/new.txt"));
3729 assert!(
3730 snapshot
3731 .current_files_for(&workspace.root, Some(&right), "test")
3732 .unwrap()
3733 .files
3734 .contains("right/empty/new.txt")
3735 );
3736 std::fs::create_dir_all(root.path().join("left/empty/nested")).unwrap();
3737 std::fs::write(root.path().join("left/empty/nested/new.txt"), "left").unwrap();
3738 assert!(
3739 snapshot
3740 .current_files_for(&workspace.root, Some(&left), "test")
3741 .unwrap()
3742 .files
3743 .contains("left/empty/nested/new.txt")
3744 );
3745 std::fs::write(root.path().join("right/empty/another.txt"), "right").unwrap();
3746 assert!(
3747 snapshot
3748 .current_files_for(&workspace.root, Some(&all), "test")
3749 .unwrap()
3750 .files
3751 .contains("right/empty/another.txt")
3752 );
3753 }
3754
3755 #[test]
3756 fn membership_watches_the_directories_json_globs_select_from() {
3757 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/dist"])));
3758 let targets = &mut workspace.projects.get_mut("app").unwrap().targets;
3759 targets.get_mut("build").unwrap().inputs = Some(vec![json!(
3760 {"json": "{workspaceRoot}/config/**/*.json", "fields": ["version"]}
3761 )]);
3762 std::fs::create_dir_all(root.path().join("config/empty")).unwrap();
3763 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
3764 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3765 snapshot.current_files(&workspace.root).unwrap();
3766 std::thread::sleep(Duration::from_millis(2100));
3767 let prefixes = snapshot
3768 .membership_prefixes(&workspace, &graph.tasks["app:build"])
3769 .unwrap();
3770 snapshot
3771 .current_files_for(&workspace.root, Some(&prefixes), "test")
3772 .unwrap();
3773 std::fs::write(
3774 root.path().join("config/empty/new.json"),
3775 r#"{"version": 1}"#,
3776 )
3777 .unwrap();
3778 assert!(
3779 snapshot
3780 .current_files_for(&workspace.root, Some(&prefixes), "test")
3781 .unwrap()
3782 .files
3783 .contains("config/empty/new.json")
3784 );
3785 }
3786
3787 #[test]
3788 fn membership_watches_git_ignored_tracked_parents_and_index_policy() {
3789 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3790 std::fs::create_dir_all(root.path().join("tracked/nested")).unwrap();
3791 std::fs::write(root.path().join("tracked/.gitignore"), "nested/\n").unwrap();
3792 std::fs::write(root.path().join("tracked/nested/source"), "one").unwrap();
3793 assert!(
3794 paths::git(root.path(), &["init", "-q"])
3795 .unwrap()
3796 .status
3797 .success()
3798 );
3799 assert!(
3800 paths::git(root.path(), &["add", "-f", "tracked/nested/source"])
3801 .unwrap()
3802 .status
3803 .success()
3804 );
3805 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3806 let files = || snapshot.current_files(root.path()).unwrap();
3807 assert!(files().contains("tracked/nested/source"));
3808 std::fs::remove_file(root.path().join("tracked/nested/source")).unwrap();
3809 assert!(!files().contains("tracked/nested/source"));
3810 std::fs::write(root.path().join("tracked/nested/another"), "two").unwrap();
3811 assert!(!files().contains("tracked/nested/another"));
3812 assert!(
3813 paths::git(root.path(), &["add", "-f", "tracked/nested/another"])
3814 .unwrap()
3815 .status
3816 .success()
3817 );
3818 assert!(files().contains("tracked/nested/another"));
3819 std::fs::write(root.path().join("untracked"), "three").unwrap();
3820 assert!(files().contains("untracked"));
3821 std::fs::write(root.path().join(".git/info/exclude"), "untracked\n").unwrap();
3822 assert!(!files().contains("untracked"));
3823 }
3824
3825 #[test]
3826 fn a_new_git_config_include_invalidates_settled_listing_policy() {
3827 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3828 let policy = tempfile::tempdir().unwrap();
3829 let included = policy.path().join("included.config");
3830 let excludes = policy.path().join("excludes");
3831 std::fs::write(&excludes, "hidden\n").unwrap();
3832 std::fs::write(root.path().join("hidden"), "source").unwrap();
3833 assert!(
3834 paths::git(root.path(), &["init", "-q"])
3835 .unwrap()
3836 .status
3837 .success()
3838 );
3839 assert!(
3840 paths::git(
3841 root.path(),
3842 &["config", "include.path", included.to_str().unwrap()]
3843 )
3844 .unwrap()
3845 .status
3846 .success()
3847 );
3848 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3849 snapshot.current_files(&workspace.root).unwrap();
3850 std::thread::sleep(Duration::from_millis(2100));
3851 let before = snapshot.current_files(&workspace.root).unwrap();
3852 assert!(before.contains("hidden"));
3853 assert!(Arc::ptr_eq(
3854 &before,
3855 &snapshot.current_files(&workspace.root).unwrap()
3856 ));
3857 assert!(
3858 paths::git(
3859 root.path(),
3860 &[
3861 "config",
3862 "--file",
3863 included.to_str().unwrap(),
3864 "core.excludesfile",
3865 excludes.to_str().unwrap(),
3866 ]
3867 )
3868 .unwrap()
3869 .status
3870 .success()
3871 );
3872 assert!(
3873 !snapshot
3874 .current_files(&workspace.root)
3875 .unwrap()
3876 .contains("hidden")
3877 );
3878 }
3879
3880 #[cfg(unix)]
3881 #[test]
3882 fn git_policy_paths_preserve_quoted_origins_and_multiline_values() {
3883 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3884 let policy = tempfile::tempdir().unwrap();
3885 let included = policy.path().join("included\\é\t\n.config");
3887 let excludes = policy.path().join("excludes\\é\t\n");
3888 let replacement = policy.path().join("replacement\n");
3889 std::fs::write(&excludes, "hidden\n").unwrap();
3890 std::fs::write(&replacement, "visible\n").unwrap();
3891 for file in ["hidden", "visible"] {
3892 std::fs::write(root.path().join(file), "source").unwrap();
3893 }
3894 let git = |args: &[&str]| {
3895 let output = paths::git(root.path(), args).unwrap();
3896 assert!(output.status.success(), "{output:?}");
3897 output
3898 };
3899 git(&["init", "-q"]);
3900 git(&["config", "include.path", included.to_str().unwrap()]);
3901 git(&[
3902 "config",
3903 "--file",
3904 included.to_str().unwrap(),
3905 "core.excludesfile",
3906 excludes.to_str().unwrap(),
3907 ]);
3908 let text = git(&["config", "--show-origin", "--list"]);
3909 let text = String::from_utf8_lossy(&text.stdout);
3910 let quoted_origin = text
3911 .lines()
3912 .find_map(|line| {
3913 line.strip_prefix("file:\"")
3914 .and_then(|_| line.split_once('\t'))
3915 })
3916 .unwrap()
3917 .0
3918 .strip_prefix("file:")
3919 .unwrap();
3920 let policies = git_membership_policies(root.path());
3921 assert!(!policies.contains(&root.path().join(quoted_origin)));
3922 assert!(policies.contains(&included));
3923 assert!(policies.contains(&excludes));
3924 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3925 let initial = snapshot.current_files(&workspace.root).unwrap();
3926 assert!(!initial.contains("hidden"));
3927 assert!(initial.contains("visible"));
3928 std::thread::sleep(Duration::from_millis(2100));
3929 let settled = snapshot.current_files(&workspace.root).unwrap();
3930 assert!(Arc::ptr_eq(
3931 &settled,
3932 &snapshot.current_files(&workspace.root).unwrap()
3933 ));
3934 git(&[
3935 "config",
3936 "--file",
3937 included.to_str().unwrap(),
3938 "core.excludesfile",
3939 replacement.to_str().unwrap(),
3940 ]);
3941 let changed = snapshot.current_files(&workspace.root).unwrap();
3942 assert!(changed.contains("hidden"));
3943 assert!(!changed.contains("visible"));
3944 assert!(git_membership_policies(root.path()).contains(&replacement));
3945 }
3946
3947 #[cfg(unix)]
3948 #[test]
3949 fn symlinked_ignore_policy_never_reuses_stale_membership() {
3950 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
3951 let policy = tempfile::NamedTempFile::new().unwrap();
3952 std::fs::write(policy.path(), "hidden\n").unwrap();
3953 std::os::unix::fs::symlink(policy.path(), root.path().join(".gitignore")).unwrap();
3954 std::fs::write(root.path().join("hidden"), "source").unwrap();
3955 let snapshot = Snapshot::new(&workspace, &graph, &workspace.root.join("cache")).unwrap();
3956 assert!(
3957 !snapshot
3958 .current_files(root.path())
3959 .unwrap()
3960 .contains("hidden")
3961 );
3962 assert!(
3963 !snapshot
3964 .membership
3965 .lock()
3966 .unwrap()
3967 .as_ref()
3968 .unwrap()
3969 .settled
3970 );
3971 std::thread::sleep(Duration::from_millis(2100));
3972 let before = snapshot.current_files(root.path()).unwrap();
3973 assert!(Arc::ptr_eq(
3974 &before,
3975 &snapshot.current_files(root.path()).unwrap()
3976 ));
3977 std::fs::write(policy.path(), "").unwrap();
3978 assert!(
3979 snapshot
3980 .current_files(root.path())
3981 .unwrap()
3982 .contains("hidden")
3983 );
3984 }
3985
3986 #[cfg(unix)]
3987 #[test]
3988 fn recent_change_time_prevents_membership_reuse_despite_old_mtime() {
3989 use std::os::unix::fs::MetadataExt;
3990 let root = tempfile::tempdir().unwrap();
3991 let old = SystemTime::UNIX_EPOCH + Duration::from_secs(1);
3992 File::open(root.path())
3993 .unwrap()
3994 .set_times(std::fs::FileTimes::new().set_modified(old))
3995 .unwrap();
3996 let metadata = std::fs::metadata(root.path()).unwrap();
3997 assert!(metadata.ctime() > 1);
3998 let stamp = Stamp::new(&metadata);
3999 assert!(stamp.settled());
4000 assert!(!stamp.membership_settled());
4001 }
4002
4003 #[test]
4004 fn inspection_keeps_deleted_candidates() {
4005 let (root, workspace, graph) = fixture(Some(json!(["{projectRoot}/dist"])));
4006 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache"))
4007 .unwrap()
4008 .with_candidates(&["src/deleted".into()]);
4009 let resolved = resolve(
4010 &snapshot,
4011 &workspace,
4012 &graph.tasks["app:post"],
4013 None,
4014 &AtomicBool::new(false),
4015 )
4016 .unwrap();
4017 assert!(resolved.declared.files.contains("src/deleted"));
4018 }
4019
4020 #[test]
4021 fn undeclared_writes_invalidate_dependency_values() {
4022 for outputs in [None, Some(json!(["{projectRoot}/generated"]))] {
4023 let (root, workspace, graph) = fixture(outputs);
4024 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
4025 let keys = || {
4026 dependency_keys(
4027 &snapshot,
4028 &workspace,
4029 &graph,
4030 &graph.tasks["app:build"],
4031 &AtomicBool::new(false),
4032 )
4033 .unwrap()
4034 };
4035 let before = keys();
4036 std::fs::write(root.path().join("generated/value"), "different").unwrap();
4037 snapshot.outputs_written(&workspace, &graph.tasks["app:post"]);
4038 assert_ne!(before, keys());
4039 }
4040 }
4041
4042 #[cfg(unix)]
4043 #[test]
4044 fn dependency_symlinks_recheck_targets_outside_output_roots() {
4045 let (root, mut workspace, _) = fixture(Some(json!(["{projectRoot}/generated"])));
4046 workspace
4047 .projects
4048 .get_mut("app")
4049 .unwrap()
4050 .targets
4051 .get_mut("gen")
4052 .unwrap()
4053 .outputs = Some(vec!["{projectRoot}/links".into()]);
4054 std::fs::create_dir(root.path().join("links")).unwrap();
4055 std::os::unix::fs::symlink("../generated/value", root.path().join("links/value")).unwrap();
4056 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
4057 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
4058 let keys = || {
4059 dependency_keys(
4060 &snapshot,
4061 &workspace,
4062 &graph,
4063 &graph.tasks["app:build"],
4064 &AtomicBool::new(false),
4065 )
4066 .unwrap()
4067 };
4068 let before = keys();
4069 std::fs::write(root.path().join("generated/value"), "different").unwrap();
4070 snapshot.outputs_written(&workspace, &graph.tasks["app:post"]);
4071 assert_ne!(before["app:gen:**/*"], keys()["app:gen:**/*"]);
4072 }
4073 #[test]
4076 fn digests_of_ignored_files_read_by_include_ignored_are_saved() {
4077 let root = tempfile::tempdir().unwrap();
4078 std::fs::write(root.path().join("nx.json"), "{}").unwrap();
4079 std::fs::write(root.path().join(".gitignore"), "generated/\n").unwrap();
4080 std::fs::write(
4081 root.path().join("project.json"),
4082 json!({"name": "app", "targets": {"build": {"command": "true", "inputs": [
4083 {"fileset": "{projectRoot}/generated/**/*", "includeIgnored": true}
4084 ]}}})
4085 .to_string(),
4086 )
4087 .unwrap();
4088 std::fs::create_dir(root.path().join("generated")).unwrap();
4089 std::fs::write(root.path().join("generated/schema.json"), "{}").unwrap();
4090 let old = std::time::SystemTime::now() - std::time::Duration::from_secs(60);
4092 std::fs::File::options()
4093 .write(true)
4094 .open(root.path().join("generated/schema.json"))
4095 .unwrap()
4096 .set_times(std::fs::FileTimes::new().set_modified(old))
4097 .unwrap();
4098 let git = |args: &[&str]| assert!(paths::git(root.path(), args).unwrap().status.success());
4099 git(&["init", "--quiet"]);
4100 let workspace = Workspace::load(root.path()).unwrap();
4101 let graph = TaskGraph::build(&workspace, &[Request::parse("app:build").unwrap()]).unwrap();
4102 let snapshot = Snapshot::new(&workspace, &graph, &root.path().join("cache")).unwrap();
4103 assert!(!snapshot.files.contains("generated/schema.json"));
4104 let prepared =
4105 qk_executor::prepare(&workspace, &graph.tasks["app:build"], &BTreeMap::new()).unwrap();
4106 inputs(
4107 &snapshot,
4108 &workspace,
4109 &graph.tasks["app:build"],
4110 &prepared,
4111 &BTreeMap::new(),
4112 &AtomicBool::new(false),
4113 )
4114 .unwrap();
4115 snapshot.save_digests(root.path());
4116 assert!(load_digests(root.path()).contains_key("generated/schema.json"));
4117 }
4118}