1use std::collections::{BTreeMap, BTreeSet};
17use std::path::{Path, PathBuf};
18use std::time::SystemTime;
19
20use anyhow::Result;
21use qk_input_analysis::{Access, Operation};
22use serde::{Deserialize, Serialize};
23use serde_json::{Map, Value, json};
24
25use crate::hash::{Snapshot, key};
26use crate::paths::Outputs;
27
28const KEPT: usize = 8;
32
33#[derive(Clone, Debug, Default, Eq, Serialize, Deserialize)]
34#[serde(rename_all = "camelCase")]
35pub struct PathSet {
36 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
38 pub reads: BTreeSet<String>,
39 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
41 pub listings: BTreeSet<String>,
42 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
46 pub present: BTreeSet<String>,
47 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
49 pub absent: BTreeSet<String>,
50 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
53 pub written: BTreeSet<String>,
54 #[serde(default, skip_serializing_if = "Option::is_none")]
58 pub recorded: Option<Values>,
59}
60
61impl PartialEq for PathSet {
64 fn eq(&self, other: &Self) -> bool {
65 (
66 &self.reads,
67 &self.listings,
68 &self.present,
69 &self.absent,
70 &self.written,
71 ) == (
72 &other.reads,
73 &other.listings,
74 &other.present,
75 &other.absent,
76 &other.written,
77 )
78 }
79}
80
81#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
84pub struct Values {
85 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
86 pub reads: BTreeMap<String, String>,
87 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
88 pub listings: BTreeMap<String, String>,
89 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
90 pub existence: BTreeMap<String, String>,
91}
92
93impl Values {
94 fn of(values: &Value) -> Result<Self> {
96 let digests = |part: &str| -> Result<BTreeMap<String, String>> {
97 values[part]
98 .as_object()
99 .into_iter()
100 .flatten()
101 .map(|(path, value)| Ok((path.clone(), key(value)?[..16].to_owned())))
102 .collect()
103 };
104 Ok(Self {
105 reads: digests("reads")?,
106 listings: digests("listings")?,
107 existence: digests("existence")?,
108 })
109 }
110}
111
112pub struct Exclusions<'a> {
114 pub roots: [&'a Path; 2],
116 pub outputs: &'a [&'a Outputs],
118 pub ignore: crate::ultracache::IgnoredReads,
120 pub state: Vec<String>,
122}
123
124impl Exclusions<'_> {
125 fn relative(&self, path: &str) -> Option<String> {
127 let path = qk_input_analysis::strace::normalize(Path::new(path));
128 let relative = self
129 .roots
130 .iter()
131 .find_map(|root| path.strip_prefix(root).ok())?;
132 let relative = relative.to_str()?.replace('\\', "/");
133 Some(if relative.is_empty() {
134 ".".into()
135 } else {
136 relative
137 })
138 }
139
140 fn excluded(&self, path: &str) -> bool {
141 let mut parts = path.split('/');
142 parts.clone().next() == Some(".git")
143 || parts.any(|part| part == "node_modules")
144 || self.state.iter().any(|state| {
145 path == state
146 || path
147 .strip_prefix(state.as_str())
148 .is_some_and(|rest| rest.starts_with('/'))
149 })
150 || self.outputs.iter().any(|outputs| outputs.matches(path))
151 || self.ignore.is_match(path)
152 }
153}
154
155impl PathSet {
156 pub fn from_accesses(accesses: &[Access], exclusions: &Exclusions, root: &Path) -> Self {
160 let mut set = Self::default();
161 let mut written = BTreeSet::new();
162 for access in accesses {
163 let Some(path) = exclusions.relative(&access.path) else {
164 continue;
165 };
166 if access.operation.writes() {
167 written.insert(path);
168 continue;
169 }
170 if written.contains(&path) || exclusions.excluded(&path) {
172 continue;
173 }
174 let found = matches!(access.result.as_str(), "success" | "unknown");
175 let missing = matches!(access.result.as_str(), "missing" | "ENOTDIR");
176 match access.operation {
177 Operation::OpenRead | Operation::ReadData | Operation::Execute if found => {
178 if !root.join(&path).is_dir() {
181 set.reads.insert(path);
182 }
183 }
184 Operation::ListDirectory | Operation::ReadDirectory if found => {
185 set.listings.insert(path);
186 }
187 Operation::Metadata | Operation::ReadLink if found => {
188 set.present.insert(path);
189 }
190 _ if missing => {
191 set.absent.insert(path);
192 }
193 _ => {}
194 }
195 }
196 set.present.retain(|path| !set.reads.contains(path));
197 set.absent
198 .retain(|path| !set.reads.contains(path) && !set.present.contains(path));
199 set.written = written
200 .into_iter()
201 .filter(|path| set.listings.contains(parent(path)))
202 .collect();
203 set
204 }
205
206 pub fn is_relative(&self) -> bool {
209 [
210 &self.reads,
211 &self.listings,
212 &self.present,
213 &self.absent,
214 &self.written,
215 ]
216 .into_iter()
217 .flatten()
218 .all(|path| {
219 !path.is_empty()
220 && Path::new(path).components().all(|component| {
221 matches!(
222 component,
223 std::path::Component::Normal(_) | std::path::Component::CurDir
224 )
225 })
226 })
227 }
228
229 pub fn is_empty(&self) -> bool {
230 self.reads.is_empty()
231 && self.listings.is_empty()
232 && self.present.is_empty()
233 && self.absent.is_empty()
234 }
235
236 pub(crate) fn changed_since(
255 &self,
256 root: &Path,
257 started: SystemTime,
258 unlisted: &Unlisted,
259 before: &Before,
260 ) -> Option<Changed<'_>> {
261 let file = self.reads.iter().chain(&self.present).find(|path| {
262 std::fs::symlink_metadata(root.join(path))
263 .is_ok_and(|metadata| !metadata.is_dir() && changed(&metadata) >= started)
264 });
265 if let Some(path) = file {
266 return Some(Changed { path, how: None });
267 }
268 self.listings.iter().find_map(|directory| {
269 self.listing_changed(root, directory, started, unlisted, before)
270 .map(|how| Changed {
271 path: directory,
272 how,
273 })
274 })
275 }
276
277 fn listing_changed(
280 &self,
281 root: &Path,
282 directory: &str,
283 started: SystemTime,
284 unlisted: &Unlisted,
285 before: &Before,
286 ) -> Option<Option<String>> {
287 if directory != "." && unlisted.matches(directory) {
290 return None;
291 }
292 let absolute = if directory == "." {
293 root.to_path_buf()
294 } else {
295 root.join(directory)
296 };
297 let skipped = |path: &str| self.written.contains(path) || unlisted.matches(path);
298 if let Some(earlier) = before.0.get(directory) {
299 let earlier: BTreeSet<_> = earlier
300 .iter()
301 .filter(|(name, _)| !self.written.contains(&child(directory, name)))
302 .cloned()
303 .collect();
304 let Some(now) = entries(root, directory, skipped) else {
305 return Some(Some("it could no longer be read".to_owned()));
306 };
307 if let Some((name, _)) = now.difference(&earlier).next() {
308 return Some(Some(format!("{name} added")));
309 }
310 if let Some((name, _)) = earlier.difference(&now).next() {
311 return Some(Some(format!("{name} removed")));
312 }
313 return None;
314 }
315 let Ok(entries) = std::fs::read_dir(&absolute) else {
316 return Some(Some("it was removed or could no longer be read".to_owned()));
317 };
318 let entries: Vec<(std::fs::DirEntry, bool, bool)> = entries
321 .flatten()
322 .map(|entry| {
323 let path = child(directory, &entry.file_name().to_string_lossy());
324 let own = self.written.contains(&path) || unlisted.own(&path);
325 (entry, skipped(&path), own)
326 })
327 .collect();
328 let new = |entry: &std::fs::DirEntry| {
332 std::fs::symlink_metadata(entry.path()).is_ok_and(|metadata| {
333 if metadata.is_dir() {
334 metadata.created().is_ok_and(|born| born >= started)
335 } else {
336 changed(&metadata) >= started
337 }
338 })
339 };
340 let unskipped = || entries.iter().filter(|(_, skipped, _)| !skipped);
341 let name = |entry: &std::fs::DirEntry| entry.file_name().to_string_lossy().into_owned();
342 let own = self.written.iter().any(|path| parent(path) == directory)
343 || entries.iter().any(|(_, _, own)| *own);
344 if !own {
345 let moved = std::fs::symlink_metadata(&absolute)
346 .and_then(|metadata| metadata.modified())
347 .is_ok_and(|modified| modified >= started);
348 return moved.then(|| {
349 Some(unskipped().find(|(entry, _, _)| new(entry)).map_or_else(
350 || "an entry removed or renamed".to_owned(),
351 |(entry, _, _)| format!("{} added or changed", name(entry)),
352 ))
353 });
354 }
355 unskipped()
356 .find(|(entry, _, _)| new(entry))
357 .map(|(entry, _, _)| Some(format!("{} added or changed", name(entry))))
358 }
359
360 pub fn digest(&self) -> Result<String> {
362 let paths = Self {
363 recorded: None,
364 ..self.clone()
365 };
366 key(&serde_json::to_value(paths)?)
367 }
368
369 pub(crate) fn changed(
373 &self,
374 snapshot: &Snapshot,
375 root: &Path,
376 unlisted: &Unlisted,
377 ) -> Result<Vec<(&str, &'static str)>> {
378 let Some(recorded) = &self.recorded else {
379 return Ok(Vec::new());
380 };
381 let now = Values::of(&self.values(snapshot, root, unlisted)?)?;
382 let differs = |then: &BTreeMap<String, String>,
383 now: &BTreeMap<String, String>,
384 path: &str| { then.get(path) != now.get(path) };
385 let mut changed = Vec::new();
386 for path in &self.reads {
387 if differs(&recorded.reads, &now.reads, path) {
388 changed.push((path.as_str(), "read"));
389 }
390 }
391 for path in &self.listings {
392 if differs(&recorded.listings, &now.listings, path) {
393 changed.push((path.as_str(), "listed"));
394 }
395 }
396 for (paths, how) in [(&self.present, "found"), (&self.absent, "looked for")] {
397 for path in paths {
398 if differs(&recorded.existence, &now.existence, path) {
399 changed.push((path.as_str(), how));
400 }
401 }
402 }
403 Ok(changed)
404 }
405
406 pub(crate) fn values(
410 &self,
411 snapshot: &Snapshot,
412 root: &Path,
413 unlisted: &Unlisted,
414 ) -> Result<Value> {
415 let value = |path: &str| match kind(&root.join(path)) {
417 "file" | "symlink" => snapshot
418 .recorded_value(root, path)
419 .unwrap_or_else(|_| json!({"unreadable": true})),
420 other => json!(other),
421 };
422 let reads: Map<String, Value> = self
423 .reads
424 .iter()
425 .map(|path| (path.clone(), value(path)))
426 .collect();
427 let mut listings = Map::new();
428 for directory in &self.listings {
429 listings.insert(directory.clone(), self.listing(root, directory, unlisted)?);
430 }
431 let existence: Map<String, Value> = self
432 .present
433 .iter()
434 .chain(&self.absent)
435 .map(|path| (path.clone(), value(path)))
436 .collect();
437 Ok(json!({"reads": reads, "listings": listings, "existence": existence}))
438 }
439
440 fn listing(&self, root: &Path, directory: &str, unlisted: &Unlisted) -> Result<Value> {
442 let absolute = if directory == "." {
443 root.to_path_buf()
444 } else {
445 root.join(directory)
446 };
447 let entries = match std::fs::read_dir(&absolute) {
448 Ok(entries) => entries,
449 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
450 return Ok(json!("absent"));
451 }
452 Err(error) if error.kind() == std::io::ErrorKind::NotADirectory => {
453 return Ok(json!(kind(&absolute)));
454 }
455 Err(error) => return Err(error.into()),
456 };
457 let mut listed = Map::new();
458 for entry in entries {
459 let entry = entry?;
460 let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
461 continue;
462 };
463 let path = if directory == "." {
464 name.clone()
465 } else {
466 format!("{directory}/{name}")
467 };
468 if self.written.contains(&path) || unlisted.matches(&path) {
469 continue;
470 }
471 listed.insert(name, json!(kind(&entry.path())));
472 }
473 Ok(Value::Object(listed))
474 }
475}
476
477fn parent(path: &str) -> &str {
479 path.rsplit_once('/').map_or(".", |(parent, _)| parent)
480}
481
482fn child(directory: &str, name: &str) -> String {
484 if directory == "." {
485 name.to_owned()
486 } else {
487 format!("{directory}/{name}")
488 }
489}
490
491fn entries(
494 root: &Path,
495 directory: &str,
496 skipped: impl Fn(&str) -> bool,
497) -> Option<BTreeSet<(String, &'static str)>> {
498 let absolute = if directory == "." {
499 root.to_path_buf()
500 } else {
501 root.join(directory)
502 };
503 let mut found = BTreeSet::new();
504 for entry in std::fs::read_dir(&absolute).ok()? {
505 let entry = entry.ok()?;
506 let name = entry.file_name().to_string_lossy().into_owned();
507 if !skipped(&child(directory, &name)) {
508 found.insert((name, file_kind(entry.file_type().ok()?)));
509 }
510 }
511 Some(found)
512}
513
514fn kind(path: &Path) -> &'static str {
516 std::fs::symlink_metadata(path).map_or("absent", |metadata| file_kind(metadata.file_type()))
517}
518
519fn file_kind(file_type: std::fs::FileType) -> &'static str {
520 if file_type.is_symlink() {
521 "symlink"
522 } else if file_type.is_dir() {
523 "directory"
524 } else if file_type.is_file() {
525 "file"
526 } else {
527 "other"
528 }
529}
530
531fn relative_root(root: &str) -> &str {
535 let mut root = root.trim_end_matches('/');
536 while let Some(rest) = root.strip_prefix("./") {
537 root = rest;
538 }
539 if root == "." { "" } else { root }
540}
541
542#[derive(Debug, PartialEq, Eq)]
545pub(crate) struct Changed<'a> {
546 pub path: &'a str,
547 pub how: Option<String>,
548}
549
550impl std::fmt::Display for Changed<'_> {
551 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
552 match &self.how {
553 Some(how) => write!(f, "{} changed while it ran ({how})", self.path),
554 None => write!(f, "{} changed while it ran", self.path),
555 }
556 }
557}
558
559#[derive(Default)]
565pub(crate) struct WorkspaceOutputs {
566 outputs: Vec<(TargetOf, Outputs)>,
567 by_anchor: BTreeMap<String, Vec<usize>>,
568 folders: Vec<Vec<String>>,
571 by_folder: BTreeMap<String, Vec<usize>>,
572}
573
574impl WorkspaceOutputs {
575 pub(crate) fn matches(&self, path: &str) -> bool {
578 self.outputs
579 .iter()
580 .any(|(_, outputs)| outputs.matches(path))
581 }
582}
583
584impl WorkspaceOutputs {
585 pub(crate) fn new(outputs: Vec<(TargetOf, Outputs)>, roots: &BTreeMap<String, String>) -> Self {
587 let mut by_anchor: BTreeMap<String, Vec<usize>> = BTreeMap::new();
588 let mut by_folder: BTreeMap<String, Vec<usize>> = BTreeMap::new();
589 let mut folders = Vec::new();
590 let projects: BTreeSet<&str> = roots
593 .values()
594 .map(|root| relative_root(root))
595 .filter(|root| !root.is_empty())
596 .flat_map(|root| {
597 std::iter::once(root).chain(root.match_indices('/').map(|(at, _)| &root[..at]))
598 })
599 .collect();
600 for (index, ((project, _, _), declared)) in outputs.iter().enumerate() {
601 let root = roots.get(project).map_or("", |root| relative_root(root));
602 let mut held = Vec::new();
603 for anchor in declared.anchors() {
604 by_anchor.entry(anchor.to_owned()).or_default().push(index);
605 let rest = if root.is_empty() {
606 Some(anchor)
607 } else {
608 anchor
609 .strip_prefix(root)
610 .and_then(|rest| rest.strip_prefix('/'))
611 };
612 let Some(rest) = rest else {
613 continue;
614 };
615 for (at, _) in rest.match_indices('/') {
617 let folder = if root.is_empty() {
618 rest[..at].to_owned()
619 } else {
620 format!("{root}/{}", &rest[..at])
621 };
622 if projects.contains(folder.as_str()) {
623 continue;
624 }
625 by_folder.entry(folder.clone()).or_default().push(index);
626 held.push(folder);
627 }
628 }
629 folders.push(held);
630 }
631 Self {
632 outputs,
633 by_anchor,
634 folders,
635 by_folder,
636 }
637 }
638
639 fn holding<'a>(&'a self, path: &'a str) -> impl Iterator<Item = &'a (TargetOf, Outputs)> {
641 std::iter::once(path)
642 .chain(path.match_indices('/').map(|(at, _)| &path[..at]))
643 .filter_map(|anchor| self.by_anchor.get(anchor))
644 .flatten()
645 .map(|&index| &self.outputs[index])
646 }
647}
648
649pub(crate) type TargetOf = (String, String, Option<String>);
652
653pub(crate) struct Others<'a> {
657 pub outputs: &'a WorkspaceOutputs,
658 pub upstream: BTreeSet<TargetOf>,
660}
661
662impl Others<'_> {
663 pub(crate) fn matches(&self, path: &str) -> bool {
670 let mut other = false;
671 let holding = self
672 .outputs
673 .holding(path)
674 .filter(|(_, outputs)| outputs.matches(path))
675 .map(|(target, _)| target);
676 let folders = self
677 .outputs
678 .by_folder
679 .get(path)
680 .into_iter()
681 .flatten()
682 .map(|&index| &self.outputs.outputs[index].0);
683 for target in holding.chain(folders) {
684 if self.upstream.contains(target) {
685 return false;
686 }
687 other = true;
688 }
689 other
690 }
691}
692
693pub(crate) struct Unlisted<'a> {
701 pub own: Vec<&'a Outputs>,
702 pub others: Option<Others<'a>>,
703 pub ignore: &'a crate::ultracache::IgnoredReads,
704}
705
706impl Unlisted<'_> {
707 fn matches(&self, path: &str) -> bool {
708 self.own(path)
709 || self
710 .others
711 .as_ref()
712 .is_some_and(|others| others.matches(path))
713 || self.ignore.is_match(path)
714 }
715
716 fn own(&self, path: &str) -> bool {
717 self.own.iter().any(|outputs| outputs.matches(path))
718 }
719
720 pub(crate) fn directories(&self, project_root: &str) -> BTreeSet<String> {
726 let project_root = match relative_root(project_root) {
727 "" => ".",
728 root => root,
729 };
730 let mut directories = BTreeSet::from([project_root.to_owned()]);
731 if let Some(others) = &self.others {
732 let all = &others.outputs;
733 for (index, (target, outputs)) in all.outputs.iter().enumerate() {
734 if others.upstream.contains(target) {
735 continue;
736 }
737 for anchor in outputs.anchors() {
738 directories.insert(anchor.to_owned());
739 directories.insert(parent(anchor).to_owned());
740 }
741 for folder in &all.folders[index] {
744 directories.insert(folder.clone());
745 directories.insert(parent(folder).to_owned());
746 }
747 }
748 }
749 directories
750 }
751}
752
753#[derive(Default)]
757pub(crate) struct Before(BTreeMap<String, BTreeSet<(String, &'static str)>>);
758
759impl Before {
760 pub(crate) fn take(root: &Path, directories: BTreeSet<String>, unlisted: &Unlisted) -> Self {
761 Self(
762 directories
763 .into_iter()
764 .filter_map(|directory| {
765 let entries = entries(root, &directory, |path| unlisted.matches(path))?;
766 Some((directory, entries))
767 })
768 .collect(),
769 )
770 }
771}
772
773#[derive(Serialize, Deserialize)]
775struct Stored {
776 version: u32,
777 weak: String,
778 sets: Vec<PathSet>,
779}
780
781fn location(cache: &Path, weak: &str) -> PathBuf {
782 cache.join("pathsets").join(format!("{weak}.json"))
783}
784
785pub(crate) fn load(cache: &Path, weak: &str) -> Vec<PathSet> {
788 std::fs::read(location(cache, weak))
789 .ok()
790 .and_then(|bytes| serde_json::from_slice::<Stored>(&bytes).ok())
791 .filter(|stored| stored.version == 1 && stored.weak == weak)
792 .map(|stored| stored.sets)
793 .unwrap_or_default()
794}
795
796pub(crate) fn touch(cache: &Path, weak: &str) {
799 if let Ok(file) = std::fs::File::options()
800 .append(true)
801 .open(location(cache, weak))
802 {
803 let _ = file.set_modified(std::time::SystemTime::now());
804 }
805}
806
807pub(crate) fn save(cache: &Path, weak: &str, set: PathSet) -> Result<()> {
809 let mut sets = load(cache, weak);
810 sets.retain(|kept| kept != &set);
811 sets.insert(0, set);
812 keep(cache, weak, sets)
813}
814
815pub(crate) fn record(cache: &Path, weak: &str) -> Option<Vec<u8>> {
818 std::fs::read(location(cache, weak)).ok()
819}
820
821pub(crate) fn within(record: &[u8], largest: usize) -> Result<Option<Vec<u8>>> {
825 if record.len() <= largest {
826 return Ok(Some(record.to_vec()));
827 }
828 let mut stored: Stored = serde_json::from_slice(record)?;
829 while stored.sets.len() > 1 {
830 stored.sets.pop();
831 let shorter = serde_json::to_vec(&stored)?;
832 if shorter.len() <= largest {
833 return Ok(Some(shorter));
834 }
835 }
836 Ok(None)
837}
838
839pub(crate) fn merge(cache: &Path, weak: &str, record: &[u8]) -> Result<Vec<PathSet>> {
844 let Some(fetched) = serde_json::from_slice::<Stored>(record)
845 .ok()
846 .filter(|stored| stored.version == 1 && stored.weak == weak)
847 else {
848 return Ok(Vec::new());
849 };
850 let mut sets = load(cache, weak);
851 let mut added = Vec::new();
852 for set in fetched.sets {
853 if sets.len() + added.len() < KEPT
854 && set.is_relative()
855 && !set.is_empty()
856 && !sets.contains(&set)
857 && !added.contains(&set)
858 {
859 added.push(set);
860 }
861 }
862 if !added.is_empty() {
863 sets.extend(added.iter().cloned());
864 keep(cache, weak, sets)?;
865 }
866 Ok(added)
867}
868
869fn keep(cache: &Path, weak: &str, mut sets: Vec<PathSet>) -> Result<()> {
870 sets.truncate(KEPT);
871 std::fs::create_dir_all(cache.join("pathsets"))?;
872 crate::record::publish(
873 cache,
874 &location(cache, weak),
875 &Stored {
876 version: 1,
877 weak: weak.to_owned(),
878 sets,
879 },
880 )
881}
882
883pub(crate) fn strong_key(
885 weak: &str,
886 set: &PathSet,
887 snapshot: &Snapshot,
888 root: &Path,
889 unlisted: &Unlisted,
890) -> Result<String> {
891 keyed(weak, set, &set.values(snapshot, root, unlisted)?)
892}
893
894fn keyed(weak: &str, set: &PathSet, values: &Value) -> Result<String> {
895 key(&json!({
896 "schema": "qk-recorded-v1",
897 "weak": weak,
898 "set": set.digest()?,
899 "values": values,
900 }))
901}
902
903pub(crate) fn with_values(
906 weak: &str,
907 mut set: PathSet,
908 snapshot: &Snapshot,
909 root: &Path,
910 unlisted: &Unlisted,
911) -> Result<(String, PathSet)> {
912 let values = set.values(snapshot, root, unlisted)?;
913 let strong = keyed(weak, &set, &values)?;
914 set.recorded = Some(Values::of(&values)?);
915 Ok((strong, set))
916}
917
918pub(crate) fn now(directory: &Path) -> SystemTime {
923 tempfile::NamedTempFile::new_in(directory)
924 .and_then(|file| file.as_file().metadata())
925 .map(|metadata| changed(&metadata))
926 .unwrap_or_else(|_| SystemTime::now())
927}
928
929fn changed(metadata: &std::fs::Metadata) -> SystemTime {
932 let modified = metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH);
933 #[cfg(unix)]
934 {
935 use std::os::unix::fs::MetadataExt;
936 let ctime = u64::try_from(metadata.ctime()).ok().map(|seconds| {
937 SystemTime::UNIX_EPOCH
938 + std::time::Duration::new(
939 seconds,
940 u32::try_from(metadata.ctime_nsec()).unwrap_or(0),
941 )
942 });
943 if let Some(ctime) = ctime {
944 return modified.max(ctime);
945 }
946 }
947 modified
948}
949
950#[cfg(test)]
951mod tests {
952 use super::*;
953 use crate::ultracache::IgnoredReads;
954
955 static IGNORE_NOTHING: IgnoredReads = IgnoredReads::none();
956
957 const NOTHING: Unlisted<'static> = Unlisted {
959 own: Vec::new(),
960 others: None,
961 ignore: &IGNORE_NOTHING,
962 };
963 use qk_input_analysis::Category;
964
965 fn access(path: &str, operation: Operation, result: &str) -> Access {
966 Access {
967 process: 1,
968 path: path.into(),
969 resolved_path: None,
970 keyed_path: None,
971 package: None,
972 operation,
973 result: result.into(),
974 category: Category::Uncovered,
975 }
976 }
977
978 #[test]
979 fn keeps_what_the_task_read_listed_and_looked_up() {
980 let temp = tempfile::tempdir().unwrap();
981 let root = temp.path().canonicalize().unwrap();
982 for directory in ["src", "icons", "dist"] {
983 std::fs::create_dir(root.join(directory)).unwrap();
984 }
985 let path = |relative: &str| root.join(relative).to_str().unwrap().to_owned();
986 let outputs = Outputs::from_paths(&["dist".into()]).unwrap();
987 let exclusions = Exclusions {
988 roots: [&root, &root],
989 outputs: &[&outputs],
990 ignore: IgnoredReads::of_globs(&["src/*.log".to_owned()]).unwrap(),
991 state: vec![".qk".into()],
992 };
993 let accesses = [
994 access(&path("src/a.ts"), Operation::OpenRead, "success"),
995 access(&path("src"), Operation::OpenRead, "success"),
996 access(&path("icons"), Operation::ListDirectory, "success"),
997 access(&path("config.local"), Operation::Metadata, "missing"),
998 access(&path("src/b.ts"), Operation::Metadata, "success"),
999 access(&path("icons/sprite.svg"), Operation::Write, "success"),
1001 access(&path("icons/sprite.svg"), Operation::OpenRead, "success"),
1002 access("/usr/lib/libc.so", Operation::OpenRead, "success"),
1004 access(
1005 &path("node_modules/x/index.js"),
1006 Operation::OpenRead,
1007 "success",
1008 ),
1009 access(&path(".git/HEAD"), Operation::OpenRead, "success"),
1010 access(&path(".qk/state.json"), Operation::OpenRead, "success"),
1011 access(&path("dist/out.js"), Operation::OpenRead, "success"),
1012 access(&path("src/debug.log"), Operation::OpenRead, "success"),
1013 access(&path("src/a.ts"), Operation::Metadata, "success"),
1015 ];
1016 let set = PathSet::from_accesses(&accesses, &exclusions, &root);
1017 assert_eq!(
1018 set,
1019 PathSet {
1020 reads: ["src/a.ts".into()].into(),
1021 listings: ["icons".into()].into(),
1022 present: ["src/b.ts".into()].into(),
1023 absent: ["config.local".into()].into(),
1024 written: ["icons/sprite.svg".into()].into(),
1025 recorded: None,
1026 }
1027 );
1028 }
1029
1030 #[test]
1031 fn values_change_with_contents_listings_and_existence() {
1032 let temp = tempfile::tempdir().unwrap();
1033 let root = temp.path().canonicalize().unwrap();
1034 std::fs::create_dir_all(root.join("icons")).unwrap();
1035 std::fs::write(root.join("a.txt"), "one").unwrap();
1036 std::fs::write(root.join("icons/a.svg"), "").unwrap();
1037 std::fs::write(root.join("nx.json"), "{}").unwrap();
1038 std::fs::write(
1039 root.join("project.json"),
1040 r#"{"name":"app","targets":{"build":{"command":"true"}}}"#,
1041 )
1042 .unwrap();
1043 let workspace = qk_config::Workspace::load(&root).unwrap();
1044 let graph = qk_taskgraph::TaskGraph::build(
1045 &workspace,
1046 &[qk_taskgraph::Request::parse("app:build").unwrap()],
1047 )
1048 .unwrap();
1049 let snapshot = Snapshot::new(&workspace, &graph, &root.join("cache")).unwrap();
1050 std::fs::write(root.join("checked.txt"), "one").unwrap();
1051 let set = PathSet {
1052 reads: ["a.txt".into()].into(),
1053 listings: ["icons".into()].into(),
1054 present: ["checked.txt".into()].into(),
1055 absent: ["optional.txt".into()].into(),
1056 written: ["icons/sprite.svg".into()].into(),
1057 recorded: None,
1058 };
1059 let outputs = Outputs::from_paths(&["icons/out".into()]).unwrap();
1060 let unlisted = Unlisted {
1061 own: vec![&outputs],
1062 others: None,
1063 ignore: &IGNORE_NOTHING,
1064 };
1065 let strong = || strong_key("weak", &set, &snapshot, &root, &unlisted).unwrap();
1066 let first = strong();
1067 let (kept_key, kept) =
1070 with_values("weak", set.clone(), &snapshot, &root, &unlisted).unwrap();
1071 assert_eq!(kept_key, first);
1072 assert_eq!(kept, set);
1073 assert_eq!(kept.digest().unwrap(), set.digest().unwrap());
1074 assert!(
1075 kept.changed(&snapshot, &root, &unlisted)
1076 .unwrap()
1077 .is_empty()
1078 );
1079 std::fs::write(root.join("a.txt"), "changed").unwrap();
1080 std::fs::write(root.join("optional.txt"), "").unwrap();
1081 assert_eq!(
1082 kept.changed(&snapshot, &root, &unlisted).unwrap(),
1083 [("a.txt", "read"), ("optional.txt", "looked for")]
1084 );
1085 assert!(set.changed(&snapshot, &root, &unlisted).unwrap().is_empty());
1086 std::fs::write(root.join("a.txt"), "one").unwrap();
1087 std::fs::remove_file(root.join("optional.txt")).unwrap();
1088 std::fs::write(root.join("icons/sprite.svg"), "").unwrap();
1091 std::fs::create_dir(root.join("icons/out")).unwrap();
1092 std::fs::write(root.join("b.txt"), "unread").unwrap();
1093 assert_eq!(strong(), first);
1094 for change in [
1095 |root: &Path| std::fs::write(root.join("icons/b.svg"), "").unwrap(),
1096 |root: &Path| std::fs::write(root.join("optional.txt"), "").unwrap(),
1097 |root: &Path| std::fs::write(root.join("a.txt"), "two").unwrap(),
1098 |root: &Path| std::fs::write(root.join("checked.txt"), "two").unwrap(),
1100 ] {
1101 let before = strong();
1102 change(&root);
1103 assert_ne!(strong(), before);
1104 }
1105 }
1106
1107 #[test]
1108 fn keeps_the_newest_sets_first() {
1109 let temp = tempfile::tempdir().unwrap();
1110 std::fs::create_dir(temp.path().join("tmp")).unwrap();
1111 let set = |read: &str| PathSet {
1112 reads: [read.to_owned()].into(),
1113 ..PathSet::default()
1114 };
1115 for index in 0..10 {
1116 save(temp.path(), "weak", set(&format!("{index}"))).unwrap();
1117 }
1118 save(temp.path(), "weak", set("5")).unwrap();
1119 let kept: Vec<String> = load(temp.path(), "weak")
1120 .into_iter()
1121 .map(|set| set.reads.into_iter().next().unwrap())
1122 .collect();
1123 assert_eq!(kept, ["5", "9", "8", "7", "6", "4", "3", "2"]);
1124 assert!(load(temp.path(), "other").is_empty());
1125 }
1126
1127 #[test]
1128 fn merges_the_sets_another_machine_recorded_behind_its_own() {
1129 let temp = tempfile::tempdir().unwrap();
1130 std::fs::create_dir(temp.path().join("tmp")).unwrap();
1131 let set = |read: &str| PathSet {
1132 reads: [read.to_owned()].into(),
1133 ..PathSet::default()
1134 };
1135 save(temp.path(), "weak", set("a")).unwrap();
1136 let record = |weak: &str, sets: Vec<PathSet>| {
1137 serde_json::to_vec(&Stored {
1138 version: 1,
1139 weak: weak.into(),
1140 sets,
1141 })
1142 .unwrap()
1143 };
1144 let fetched = record(
1145 "weak",
1146 vec![set("b"), set("a"), set("../escape"), set("/abs")],
1147 );
1148 assert_eq!(merge(temp.path(), "weak", &fetched).unwrap(), [set("b")]);
1149 assert_eq!(load(temp.path(), "weak"), [set("a"), set("b")]);
1150 assert!(merge(temp.path(), "weak", &fetched).unwrap().is_empty());
1151 assert!(
1152 merge(temp.path(), "weak", &record("other", vec![set("c")]))
1153 .unwrap()
1154 .is_empty()
1155 );
1156 let many = (0..10).map(|index| set(&format!("{index}"))).collect();
1157 assert_eq!(
1158 merge(temp.path(), "weak", &record("weak", many))
1159 .unwrap()
1160 .len(),
1161 6
1162 );
1163 assert_eq!(load(temp.path(), "weak").len(), KEPT);
1164 }
1165
1166 #[test]
1167 fn a_record_too_large_leaves_out_its_oldest_sets() {
1168 let set = |read: &str| PathSet {
1169 reads: [read.to_owned()].into(),
1170 ..PathSet::default()
1171 };
1172 let record = |sets: Vec<PathSet>| {
1173 serde_json::to_vec(&Stored {
1174 version: 1,
1175 weak: "weak".into(),
1176 sets,
1177 })
1178 .unwrap()
1179 };
1180 let newest = "n".repeat(100);
1181 let full = record(vec![
1182 set(&newest),
1183 set(&"o".repeat(100)),
1184 set(&"p".repeat(100)),
1185 ]);
1186 assert_eq!(within(&full, full.len()).unwrap(), Some(full.clone()));
1187 let one = record(vec![set(&newest)]);
1188 assert_eq!(within(&full, one.len() + 50).unwrap(), Some(one.clone()));
1189 assert_eq!(within(&full, one.len() - 1).unwrap(), None);
1190 }
1191
1192 #[test]
1193 fn finds_what_changed_while_the_task_ran() {
1194 let temp = tempfile::tempdir().unwrap();
1195 for path in ["read.txt", "found.txt", "listed.txt", "before.txt"] {
1196 std::fs::write(temp.path().join(path), "before").unwrap();
1197 }
1198 for directory in ["listed", "written"] {
1199 std::fs::create_dir(temp.path().join(directory)).unwrap();
1200 std::fs::write(temp.path().join(directory).join("kept.txt"), "").unwrap();
1201 }
1202 std::fs::create_dir(temp.path().join("listed/sub")).unwrap();
1203 let set = PathSet {
1204 reads: ["read.txt".to_owned(), "gone.txt".to_owned()].into(),
1205 present: ["found.txt".to_owned(), ".".to_owned()].into(),
1206 listings: ["listed".to_owned(), "written".to_owned()].into(),
1207 written: ["written/new.txt".to_owned()].into(),
1208 ..PathSet::default()
1209 };
1210 std::thread::sleep(std::time::Duration::from_millis(20));
1211 let started = now(temp.path());
1212 let changed = |set: &PathSet| {
1213 set.changed_since(temp.path(), started, &NOTHING, &Before::default())
1214 .map(|changed| changed.path.to_owned())
1215 };
1216 assert_eq!(changed(&set), None);
1217 std::fs::write(temp.path().join("written/new.txt"), "").unwrap();
1219 std::fs::write(temp.path().join("listed/kept.txt"), "after").unwrap();
1220 std::fs::write(temp.path().join("listed.txt"), "after").unwrap();
1221 let unlisted = PathSet {
1223 listings: ["written".to_owned()].into(),
1224 ..set.clone()
1225 };
1226 assert_eq!(changed(&unlisted), None);
1227 std::fs::write(temp.path().join("found.txt"), "after").unwrap();
1228 assert_eq!(changed(&set), Some("found.txt".into()));
1229 let listings = PathSet {
1230 reads: [].into(),
1231 present: [].into(),
1232 ..set.clone()
1233 };
1234 std::fs::write(temp.path().join("listed/sub/made.txt"), "").unwrap();
1237 assert_eq!(changed(&listings), None);
1238 std::fs::write(temp.path().join("written/other.txt"), "").unwrap();
1240 if std::fs::metadata(temp.path().join("written/other.txt"))
1241 .and_then(|metadata| metadata.created())
1242 .is_ok()
1243 {
1244 assert_eq!(changed(&listings), Some("written".into()));
1245 }
1246 std::fs::remove_file(temp.path().join("written/other.txt")).unwrap();
1247 #[cfg(unix)]
1250 {
1251 std::fs::rename(
1252 temp.path().join("before.txt"),
1253 temp.path().join("written/moved.txt"),
1254 )
1255 .unwrap();
1256 assert_eq!(changed(&listings), Some("written".into()));
1257 std::fs::rename(
1258 temp.path().join("written/moved.txt"),
1259 temp.path().join("before.txt"),
1260 )
1261 .unwrap();
1262 }
1263 std::fs::write(temp.path().join("listed/new.txt"), "").unwrap();
1265 assert_eq!(changed(&listings), Some("listed".into()));
1266 let removed = tempfile::tempdir().unwrap();
1268 std::fs::write(removed.path().join("gone.txt"), "").unwrap();
1269 std::thread::sleep(std::time::Duration::from_millis(20));
1270 let started = now(temp.path());
1273 let root = PathSet {
1274 listings: [".".to_owned()].into(),
1275 ..PathSet::default()
1276 };
1277 assert_eq!(
1278 root.changed_since(removed.path(), started, &NOTHING, &Before::default())
1279 .map(|changed| changed.path),
1280 None
1281 );
1282 std::fs::remove_file(removed.path().join("gone.txt")).unwrap();
1283 assert_eq!(
1284 root.changed_since(removed.path(), started, &NOTHING, &Before::default())
1285 .map(|changed| changed.path),
1286 Some(".")
1287 );
1288 }
1289
1290 #[test]
1291 fn listings_leave_out_other_targets_outputs_and_ignored_reads() {
1292 let temp = tempfile::tempdir().unwrap();
1293 let root = &temp.path().canonicalize().unwrap();
1294 std::fs::create_dir_all(root.join("app/src")).unwrap();
1295 std::fs::write(root.join("app/src/main.ts"), "").unwrap();
1296 std::fs::write(root.join("nx.json"), "{}").unwrap();
1297 std::fs::write(
1298 root.join("project.json"),
1299 r#"{"name":"app","targets":{"build":{"command":"true"}}}"#,
1300 )
1301 .unwrap();
1302 let workspace = qk_config::Workspace::load(root).unwrap();
1303 let graph = qk_taskgraph::TaskGraph::build(
1304 &workspace,
1305 &[qk_taskgraph::Request::parse("app:build").unwrap()],
1306 )
1307 .unwrap();
1308 let set = PathSet {
1309 listings: ["app".to_owned()].into(),
1310 ..PathSet::default()
1311 };
1312 std::fs::write(root.join("app/kept.txt"), "").unwrap();
1313 std::fs::create_dir(root.join("app/out")).unwrap();
1314 let roots = [("app".to_owned(), "app".to_owned())].into();
1315 let outputs = WorkspaceOutputs::new(
1316 vec![
1317 (
1318 ("app".into(), "tsc".into(), None),
1319 Outputs::from_patterns(&["app/.tsc"]),
1320 ),
1321 (
1322 ("app".into(), "codegen".into(), None),
1323 Outputs::from_patterns(&["app/generated"]),
1324 ),
1325 (
1327 ("app".into(), "bundle".into(), None),
1328 Outputs::from_patterns(&["app/out/*.js"]),
1329 ),
1330 (
1332 ("app".into(), "package".into(), None),
1333 Outputs::from_patterns(&["app/generated"]),
1334 ),
1335 (
1337 ("app".into(), "deploy".into(), None),
1338 Outputs::from_patterns(&["app/.cache/deploy"]),
1339 ),
1340 ],
1341 &roots,
1342 );
1343 let ignore = IgnoredReads::of_globs(&["**/.fallow".to_owned()]).unwrap();
1344 let unlisted = Unlisted {
1345 own: Vec::new(),
1346 others: Some(Others {
1347 outputs: &outputs,
1348 upstream: [
1350 ("app".into(), "build".into(), None),
1351 ("app".into(), "codegen".into(), None),
1352 ]
1353 .into(),
1354 }),
1355 ignore: &ignore,
1356 };
1357 assert_eq!(
1358 unlisted.directories("app"),
1359 [
1360 "app".to_owned(),
1361 "app/.cache".to_owned(),
1362 "app/.cache/deploy".to_owned(),
1363 "app/.tsc".to_owned(),
1364 "app/generated".to_owned(),
1365 "app/out".to_owned()
1366 ]
1367 .into()
1368 );
1369 let snapshot = Snapshot::new(&workspace, &graph, &root.join("cache")).unwrap();
1370 let key = || strong_key("weak", &set, &snapshot, root, &unlisted).unwrap();
1371 let before = key();
1372 std::thread::sleep(std::time::Duration::from_millis(20));
1373 let taken = Before::take(root, unlisted.directories("app"), &unlisted);
1374 let started = now(root);
1375 let changed = |taken: &Before| {
1376 set.changed_since(root, started, &unlisted, taken)
1377 .map(|changed| changed.path)
1378 };
1379 std::fs::create_dir(root.join("app/.tsc")).unwrap();
1382 std::fs::create_dir_all(root.join("app/.cache/deploy")).unwrap();
1383 std::fs::create_dir(root.join("app/.fallow")).unwrap();
1384 assert_eq!(key(), before);
1385 assert_eq!(changed(&taken), None);
1386 std::fs::write(root.join("app/.tsc/index.js"), "").unwrap();
1389 let output = PathSet {
1390 listings: ["app/.tsc".to_owned()].into(),
1391 ..PathSet::default()
1392 };
1393 assert_eq!(
1394 output.changed_since(root, started, &unlisted, &Before::default()),
1395 None
1396 );
1397 assert!(!unlisted.matches("app"));
1399 std::fs::create_dir(root.join("app/gone")).unwrap();
1401 let gone = PathSet {
1402 listings: ["app/gone".to_owned()].into(),
1403 ..PathSet::default()
1404 };
1405 std::fs::remove_dir(root.join("app/gone")).unwrap();
1406 assert_eq!(
1407 gone.changed_since(root, started, &unlisted, &Before::default())
1408 .map(|changed| changed.path),
1409 Some("app/gone")
1410 );
1411 std::fs::write(root.join("app/out/new.js"), "").unwrap();
1413 let out = PathSet {
1414 listings: ["app/out".to_owned()].into(),
1415 ..PathSet::default()
1416 };
1417 assert_eq!(out.changed_since(root, started, &unlisted, &taken), None);
1418 assert_eq!(
1419 out.changed_since(root, started, &unlisted, &Before::default())
1420 .map(|changed| changed.path),
1421 Some("app/out")
1422 );
1423 assert!(!unlisted.matches("app/generated"));
1425 assert!(unlisted.matches("app/.tsc/index.js"));
1426 assert_eq!(changed(&Before::default()), Some("app"));
1429 std::fs::remove_file(root.join("app/kept.txt")).unwrap();
1431 assert_ne!(key(), before);
1432 assert_eq!(
1433 set.changed_since(root, started, &unlisted, &taken)
1434 .and_then(|changed| changed.how),
1435 Some("kept.txt removed".to_owned())
1436 );
1437 std::fs::write(root.join("app/kept.txt"), "").unwrap();
1438 assert_eq!(changed(&taken), None);
1439 std::fs::write(root.join("app/README.md"), "").unwrap();
1441 assert_ne!(key(), before);
1442 assert_eq!(
1443 set.changed_since(root, started, &unlisted, &taken)
1444 .map(|changed| changed.to_string()),
1445 Some("app changed while it ran (README.md added)".to_owned())
1446 );
1447 }
1448}