1use std::collections::BTreeSet;
17use std::path::{Path, PathBuf};
18use std::time::SystemTime;
19
20use anyhow::Result;
21use qk_input_analysis::{Access, Operation};
22use serde::{Deserialize, Serialize};
23use serde_json::{Map, Value, json};
24
25use crate::glob::Pattern;
26use crate::hash::{Snapshot, key};
27use crate::paths::Outputs;
28
29const KEPT: usize = 8;
33
34#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
35#[serde(rename_all = "camelCase")]
36pub struct PathSet {
37 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
39 pub reads: BTreeSet<String>,
40 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
42 pub listings: BTreeSet<String>,
43 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
47 pub present: BTreeSet<String>,
48 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
50 pub absent: BTreeSet<String>,
51 #[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
54 pub written: BTreeSet<String>,
55}
56
57pub struct Exclusions<'a> {
59 pub roots: [&'a Path; 2],
61 pub outputs: &'a [&'a Outputs],
63 pub ignore: Vec<Pattern>,
65 pub state: Vec<String>,
67}
68
69impl Exclusions<'_> {
70 fn relative(&self, path: &str) -> Option<String> {
72 let path = qk_input_analysis::strace::normalize(Path::new(path));
73 let relative = self
74 .roots
75 .iter()
76 .find_map(|root| path.strip_prefix(root).ok())?;
77 let relative = relative.to_str()?.replace('\\', "/");
78 Some(if relative.is_empty() {
79 ".".into()
80 } else {
81 relative
82 })
83 }
84
85 fn excluded(&self, path: &str) -> bool {
86 let mut parts = path.split('/');
87 parts.clone().next() == Some(".git")
88 || parts.any(|part| part == "node_modules")
89 || self.state.iter().any(|state| {
90 path == state
91 || path
92 .strip_prefix(state.as_str())
93 .is_some_and(|rest| rest.starts_with('/'))
94 })
95 || self.outputs.iter().any(|outputs| outputs.matches(path))
96 || self.ignore.iter().any(|pattern| pattern.is_match(path))
97 }
98}
99
100impl PathSet {
101 pub fn from_accesses(accesses: &[Access], exclusions: &Exclusions, root: &Path) -> Self {
105 let mut set = Self::default();
106 let mut written = BTreeSet::new();
107 for access in accesses {
108 let Some(path) = exclusions.relative(&access.path) else {
109 continue;
110 };
111 if access.operation.writes() {
112 written.insert(path);
113 continue;
114 }
115 if written.contains(&path) || exclusions.excluded(&path) {
117 continue;
118 }
119 let found = matches!(access.result.as_str(), "success" | "unknown");
120 let missing = matches!(access.result.as_str(), "missing" | "ENOTDIR");
121 match access.operation {
122 Operation::OpenRead | Operation::ReadData | Operation::Execute if found => {
123 if !root.join(&path).is_dir() {
126 set.reads.insert(path);
127 }
128 }
129 Operation::ListDirectory | Operation::ReadDirectory if found => {
130 set.listings.insert(path);
131 }
132 Operation::Metadata | Operation::ReadLink if found => {
133 set.present.insert(path);
134 }
135 _ if missing => {
136 set.absent.insert(path);
137 }
138 _ => {}
139 }
140 }
141 set.present.retain(|path| !set.reads.contains(path));
142 set.absent
143 .retain(|path| !set.reads.contains(path) && !set.present.contains(path));
144 set.written = written
145 .into_iter()
146 .filter(|path| set.listings.contains(parent(path)))
147 .collect();
148 set
149 }
150
151 pub fn is_relative(&self) -> bool {
154 [
155 &self.reads,
156 &self.listings,
157 &self.present,
158 &self.absent,
159 &self.written,
160 ]
161 .into_iter()
162 .flatten()
163 .all(|path| {
164 !path.is_empty()
165 && Path::new(path).components().all(|component| {
166 matches!(
167 component,
168 std::path::Component::Normal(_) | std::path::Component::CurDir
169 )
170 })
171 })
172 }
173
174 pub fn is_empty(&self) -> bool {
175 self.reads.is_empty()
176 && self.listings.is_empty()
177 && self.present.is_empty()
178 && self.absent.is_empty()
179 }
180
181 pub fn changed_since(
197 &self,
198 root: &Path,
199 started: SystemTime,
200 outputs: &[&Outputs],
201 ) -> Option<&str> {
202 let file = self.reads.iter().chain(&self.present).find(|path| {
203 std::fs::symlink_metadata(root.join(path))
204 .is_ok_and(|metadata| !metadata.is_dir() && changed(&metadata) >= started)
205 });
206 file.or_else(|| {
207 self.listings.iter().find(|directory| {
208 let absolute = if *directory == "." {
209 root.to_path_buf()
210 } else {
211 root.join(directory)
212 };
213 let Ok(entries) = std::fs::read_dir(&absolute) else {
214 return false;
215 };
216 let entries: Vec<(std::fs::DirEntry, bool)> = entries
218 .flatten()
219 .map(|entry| {
220 let name = entry.file_name().to_string_lossy().into_owned();
221 let path = if *directory == "." {
222 name
223 } else {
224 format!("{directory}/{name}")
225 };
226 let own = self.written.contains(&path)
227 || outputs.iter().any(|outputs| outputs.matches(&path));
228 (entry, own)
229 })
230 .collect();
231 let own = self.written.iter().any(|path| parent(path) == *directory)
232 || entries.iter().any(|(_, own)| *own);
233 if !own {
234 return std::fs::symlink_metadata(&absolute)
235 .and_then(|metadata| metadata.modified())
236 .is_ok_and(|modified| modified >= started);
237 }
238 entries.iter().filter(|(_, own)| !own).any(|(entry, _)| {
239 std::fs::symlink_metadata(entry.path()).is_ok_and(|metadata| {
240 if metadata.is_dir() {
245 metadata.created().is_ok_and(|born| born >= started)
246 } else {
247 changed(&metadata) >= started
248 }
249 })
250 })
251 })
252 })
253 .map(String::as_str)
254 }
255
256 pub fn digest(&self) -> Result<String> {
257 key(&serde_json::to_value(self)?)
258 }
259
260 pub(crate) fn values(
264 &self,
265 snapshot: &Snapshot,
266 root: &Path,
267 outputs: &[&Outputs],
268 ) -> Result<Value> {
269 let value = |path: &str| match kind(&root.join(path)) {
271 "file" | "symlink" => snapshot
272 .recorded_value(root, path)
273 .unwrap_or_else(|_| json!({"unreadable": true})),
274 other => json!(other),
275 };
276 let reads: Map<String, Value> = self
277 .reads
278 .iter()
279 .map(|path| (path.clone(), value(path)))
280 .collect();
281 let mut listings = Map::new();
282 for directory in &self.listings {
283 listings.insert(directory.clone(), self.listing(root, directory, outputs)?);
284 }
285 let existence: Map<String, Value> = self
286 .present
287 .iter()
288 .chain(&self.absent)
289 .map(|path| (path.clone(), value(path)))
290 .collect();
291 Ok(json!({"reads": reads, "listings": listings, "existence": existence}))
292 }
293
294 fn listing(&self, root: &Path, directory: &str, outputs: &[&Outputs]) -> Result<Value> {
296 let absolute = if directory == "." {
297 root.to_path_buf()
298 } else {
299 root.join(directory)
300 };
301 let entries = match std::fs::read_dir(&absolute) {
302 Ok(entries) => entries,
303 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
304 return Ok(json!("absent"));
305 }
306 Err(error) if error.kind() == std::io::ErrorKind::NotADirectory => {
307 return Ok(json!(kind(&absolute)));
308 }
309 Err(error) => return Err(error.into()),
310 };
311 let mut listed = Map::new();
312 for entry in entries {
313 let entry = entry?;
314 let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
315 continue;
316 };
317 let path = if directory == "." {
318 name.clone()
319 } else {
320 format!("{directory}/{name}")
321 };
322 if self.written.contains(&path) || outputs.iter().any(|outputs| outputs.matches(&path))
323 {
324 continue;
325 }
326 listed.insert(name, json!(kind(&entry.path())));
327 }
328 Ok(Value::Object(listed))
329 }
330}
331
332fn parent(path: &str) -> &str {
334 path.rsplit_once('/').map_or(".", |(parent, _)| parent)
335}
336
337fn kind(path: &Path) -> &'static str {
339 match std::fs::symlink_metadata(path) {
340 Ok(metadata) if metadata.file_type().is_symlink() => "symlink",
341 Ok(metadata) if metadata.is_dir() => "directory",
342 Ok(metadata) if metadata.is_file() => "file",
343 Ok(_) => "other",
344 Err(_) => "absent",
345 }
346}
347
348#[derive(Serialize, Deserialize)]
350struct Stored {
351 version: u32,
352 weak: String,
353 sets: Vec<PathSet>,
354}
355
356fn location(cache: &Path, weak: &str) -> PathBuf {
357 cache.join("pathsets").join(format!("{weak}.json"))
358}
359
360pub(crate) fn load(cache: &Path, weak: &str) -> Vec<PathSet> {
363 std::fs::read(location(cache, weak))
364 .ok()
365 .and_then(|bytes| serde_json::from_slice::<Stored>(&bytes).ok())
366 .filter(|stored| stored.version == 1 && stored.weak == weak)
367 .map(|stored| stored.sets)
368 .unwrap_or_default()
369}
370
371pub(crate) fn touch(cache: &Path, weak: &str) {
374 if let Ok(file) = std::fs::File::options()
375 .append(true)
376 .open(location(cache, weak))
377 {
378 let _ = file.set_modified(std::time::SystemTime::now());
379 }
380}
381
382pub(crate) fn save(cache: &Path, weak: &str, set: PathSet) -> Result<()> {
384 let mut sets = load(cache, weak);
385 sets.retain(|kept| kept != &set);
386 sets.insert(0, set);
387 keep(cache, weak, sets)
388}
389
390pub(crate) fn record(cache: &Path, weak: &str) -> Option<Vec<u8>> {
393 std::fs::read(location(cache, weak)).ok()
394}
395
396pub(crate) fn merge(cache: &Path, weak: &str, record: &[u8]) -> Result<Vec<PathSet>> {
401 let Some(fetched) = serde_json::from_slice::<Stored>(record)
402 .ok()
403 .filter(|stored| stored.version == 1 && stored.weak == weak)
404 else {
405 return Ok(Vec::new());
406 };
407 let mut sets = load(cache, weak);
408 let mut added = Vec::new();
409 for set in fetched.sets {
410 if sets.len() + added.len() < KEPT
411 && set.is_relative()
412 && !set.is_empty()
413 && !sets.contains(&set)
414 && !added.contains(&set)
415 {
416 added.push(set);
417 }
418 }
419 if !added.is_empty() {
420 sets.extend(added.iter().cloned());
421 keep(cache, weak, sets)?;
422 }
423 Ok(added)
424}
425
426fn keep(cache: &Path, weak: &str, mut sets: Vec<PathSet>) -> Result<()> {
427 sets.truncate(KEPT);
428 std::fs::create_dir_all(cache.join("pathsets"))?;
429 crate::record::publish(
430 cache,
431 &location(cache, weak),
432 &Stored {
433 version: 1,
434 weak: weak.to_owned(),
435 sets,
436 },
437 )
438}
439
440pub(crate) fn strong_key(
442 weak: &str,
443 set: &PathSet,
444 snapshot: &Snapshot,
445 root: &Path,
446 outputs: &[&Outputs],
447) -> Result<String> {
448 key(&json!({
449 "schema": "qk-recorded-v1",
450 "weak": weak,
451 "set": set.digest()?,
452 "values": set.values(snapshot, root, outputs)?,
453 }))
454}
455
456pub(crate) fn now(directory: &Path) -> SystemTime {
461 tempfile::NamedTempFile::new_in(directory)
462 .and_then(|file| file.as_file().metadata())
463 .map(|metadata| changed(&metadata))
464 .unwrap_or_else(|_| SystemTime::now())
465}
466
467fn changed(metadata: &std::fs::Metadata) -> SystemTime {
470 let modified = metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH);
471 #[cfg(unix)]
472 {
473 use std::os::unix::fs::MetadataExt;
474 let ctime = u64::try_from(metadata.ctime()).ok().map(|seconds| {
475 SystemTime::UNIX_EPOCH
476 + std::time::Duration::new(
477 seconds,
478 u32::try_from(metadata.ctime_nsec()).unwrap_or(0),
479 )
480 });
481 if let Some(ctime) = ctime {
482 return modified.max(ctime);
483 }
484 }
485 modified
486}
487
488#[cfg(test)]
489mod tests {
490 use super::*;
491 use qk_input_analysis::Category;
492
493 fn access(path: &str, operation: Operation, result: &str) -> Access {
494 Access {
495 process: 1,
496 path: path.into(),
497 resolved_path: None,
498 keyed_path: None,
499 package: None,
500 operation,
501 result: result.into(),
502 category: Category::Uncovered,
503 }
504 }
505
506 #[test]
507 fn keeps_what_the_task_read_listed_and_looked_up() {
508 let temp = tempfile::tempdir().unwrap();
509 let root = temp.path().canonicalize().unwrap();
510 for directory in ["src", "icons", "dist"] {
511 std::fs::create_dir(root.join(directory)).unwrap();
512 }
513 let path = |relative: &str| root.join(relative).to_str().unwrap().to_owned();
514 let outputs = Outputs::from_paths(&["dist".into()]).unwrap();
515 let exclusions = Exclusions {
516 roots: [&root, &root],
517 outputs: &[&outputs],
518 ignore: vec![Pattern::new("src/*.log", false).unwrap()],
519 state: vec![".qk".into()],
520 };
521 let accesses = [
522 access(&path("src/a.ts"), Operation::OpenRead, "success"),
523 access(&path("src"), Operation::OpenRead, "success"),
524 access(&path("icons"), Operation::ListDirectory, "success"),
525 access(&path("config.local"), Operation::Metadata, "missing"),
526 access(&path("src/b.ts"), Operation::Metadata, "success"),
527 access(&path("icons/sprite.svg"), Operation::Write, "success"),
529 access(&path("icons/sprite.svg"), Operation::OpenRead, "success"),
530 access("/usr/lib/libc.so", Operation::OpenRead, "success"),
532 access(
533 &path("node_modules/x/index.js"),
534 Operation::OpenRead,
535 "success",
536 ),
537 access(&path(".git/HEAD"), Operation::OpenRead, "success"),
538 access(&path(".qk/state.json"), Operation::OpenRead, "success"),
539 access(&path("dist/out.js"), Operation::OpenRead, "success"),
540 access(&path("src/debug.log"), Operation::OpenRead, "success"),
541 access(&path("src/a.ts"), Operation::Metadata, "success"),
543 ];
544 let set = PathSet::from_accesses(&accesses, &exclusions, &root);
545 assert_eq!(
546 set,
547 PathSet {
548 reads: ["src/a.ts".into()].into(),
549 listings: ["icons".into()].into(),
550 present: ["src/b.ts".into()].into(),
551 absent: ["config.local".into()].into(),
552 written: ["icons/sprite.svg".into()].into(),
553 }
554 );
555 }
556
557 #[test]
558 fn values_change_with_contents_listings_and_existence() {
559 let temp = tempfile::tempdir().unwrap();
560 let root = temp.path().canonicalize().unwrap();
561 std::fs::create_dir_all(root.join("icons")).unwrap();
562 std::fs::write(root.join("a.txt"), "one").unwrap();
563 std::fs::write(root.join("icons/a.svg"), "").unwrap();
564 std::fs::write(root.join("nx.json"), "{}").unwrap();
565 std::fs::write(
566 root.join("project.json"),
567 r#"{"name":"app","targets":{"build":{"command":"true"}}}"#,
568 )
569 .unwrap();
570 let workspace = qk_config::Workspace::load(&root).unwrap();
571 let graph = qk_taskgraph::TaskGraph::build(
572 &workspace,
573 &[qk_taskgraph::Request::parse("app:build").unwrap()],
574 )
575 .unwrap();
576 let snapshot = Snapshot::new(&workspace, &graph, &root.join("cache")).unwrap();
577 std::fs::write(root.join("checked.txt"), "one").unwrap();
578 let set = PathSet {
579 reads: ["a.txt".into()].into(),
580 listings: ["icons".into()].into(),
581 present: ["checked.txt".into()].into(),
582 absent: ["optional.txt".into()].into(),
583 written: ["icons/sprite.svg".into()].into(),
584 };
585 let outputs = Outputs::from_paths(&["icons/out".into()]).unwrap();
586 let strong = || strong_key("weak", &set, &snapshot, &root, &[&outputs]).unwrap();
587 let first = strong();
588 std::fs::write(root.join("icons/sprite.svg"), "").unwrap();
591 std::fs::create_dir(root.join("icons/out")).unwrap();
592 std::fs::write(root.join("b.txt"), "unread").unwrap();
593 assert_eq!(strong(), first);
594 for change in [
595 |root: &Path| std::fs::write(root.join("icons/b.svg"), "").unwrap(),
596 |root: &Path| std::fs::write(root.join("optional.txt"), "").unwrap(),
597 |root: &Path| std::fs::write(root.join("a.txt"), "two").unwrap(),
598 |root: &Path| std::fs::write(root.join("checked.txt"), "two").unwrap(),
600 ] {
601 let before = strong();
602 change(&root);
603 assert_ne!(strong(), before);
604 }
605 }
606
607 #[test]
608 fn keeps_the_newest_sets_first() {
609 let temp = tempfile::tempdir().unwrap();
610 std::fs::create_dir(temp.path().join("tmp")).unwrap();
611 let set = |read: &str| PathSet {
612 reads: [read.to_owned()].into(),
613 ..PathSet::default()
614 };
615 for index in 0..10 {
616 save(temp.path(), "weak", set(&format!("{index}"))).unwrap();
617 }
618 save(temp.path(), "weak", set("5")).unwrap();
619 let kept: Vec<String> = load(temp.path(), "weak")
620 .into_iter()
621 .map(|set| set.reads.into_iter().next().unwrap())
622 .collect();
623 assert_eq!(kept, ["5", "9", "8", "7", "6", "4", "3", "2"]);
624 assert!(load(temp.path(), "other").is_empty());
625 }
626
627 #[test]
628 fn merges_the_sets_another_machine_recorded_behind_its_own() {
629 let temp = tempfile::tempdir().unwrap();
630 std::fs::create_dir(temp.path().join("tmp")).unwrap();
631 let set = |read: &str| PathSet {
632 reads: [read.to_owned()].into(),
633 ..PathSet::default()
634 };
635 save(temp.path(), "weak", set("a")).unwrap();
636 let record = |weak: &str, sets: Vec<PathSet>| {
637 serde_json::to_vec(&Stored {
638 version: 1,
639 weak: weak.into(),
640 sets,
641 })
642 .unwrap()
643 };
644 let fetched = record(
645 "weak",
646 vec![set("b"), set("a"), set("../escape"), set("/abs")],
647 );
648 assert_eq!(merge(temp.path(), "weak", &fetched).unwrap(), [set("b")]);
649 assert_eq!(load(temp.path(), "weak"), [set("a"), set("b")]);
650 assert!(merge(temp.path(), "weak", &fetched).unwrap().is_empty());
651 assert!(
652 merge(temp.path(), "weak", &record("other", vec![set("c")]))
653 .unwrap()
654 .is_empty()
655 );
656 let many = (0..10).map(|index| set(&format!("{index}"))).collect();
657 assert_eq!(
658 merge(temp.path(), "weak", &record("weak", many))
659 .unwrap()
660 .len(),
661 6
662 );
663 assert_eq!(load(temp.path(), "weak").len(), KEPT);
664 }
665
666 #[test]
667 fn finds_what_changed_while_the_task_ran() {
668 let temp = tempfile::tempdir().unwrap();
669 for path in ["read.txt", "found.txt", "listed.txt", "before.txt"] {
670 std::fs::write(temp.path().join(path), "before").unwrap();
671 }
672 for directory in ["listed", "written"] {
673 std::fs::create_dir(temp.path().join(directory)).unwrap();
674 std::fs::write(temp.path().join(directory).join("kept.txt"), "").unwrap();
675 }
676 std::fs::create_dir(temp.path().join("listed/sub")).unwrap();
677 let set = PathSet {
678 reads: ["read.txt".to_owned(), "gone.txt".to_owned()].into(),
679 present: ["found.txt".to_owned(), ".".to_owned()].into(),
680 listings: ["listed".to_owned(), "written".to_owned()].into(),
681 written: ["written/new.txt".to_owned()].into(),
682 ..PathSet::default()
683 };
684 std::thread::sleep(std::time::Duration::from_millis(20));
685 let started = now(temp.path());
686 let changed = |set: &PathSet| {
687 set.changed_since(temp.path(), started, &[])
688 .map(str::to_owned)
689 };
690 assert_eq!(changed(&set), None);
691 std::fs::write(temp.path().join("written/new.txt"), "").unwrap();
693 std::fs::write(temp.path().join("listed/kept.txt"), "after").unwrap();
694 std::fs::write(temp.path().join("listed.txt"), "after").unwrap();
695 let unlisted = PathSet {
697 listings: ["written".to_owned()].into(),
698 ..set.clone()
699 };
700 assert_eq!(changed(&unlisted), None);
701 std::fs::write(temp.path().join("found.txt"), "after").unwrap();
702 assert_eq!(changed(&set), Some("found.txt".into()));
703 let listings = PathSet {
704 reads: [].into(),
705 present: [].into(),
706 ..set.clone()
707 };
708 std::fs::write(temp.path().join("listed/sub/made.txt"), "").unwrap();
711 assert_eq!(changed(&listings), None);
712 std::fs::write(temp.path().join("written/other.txt"), "").unwrap();
714 if std::fs::metadata(temp.path().join("written/other.txt"))
715 .and_then(|metadata| metadata.created())
716 .is_ok()
717 {
718 assert_eq!(changed(&listings), Some("written".into()));
719 }
720 std::fs::remove_file(temp.path().join("written/other.txt")).unwrap();
721 #[cfg(unix)]
724 {
725 std::fs::rename(
726 temp.path().join("before.txt"),
727 temp.path().join("written/moved.txt"),
728 )
729 .unwrap();
730 assert_eq!(changed(&listings), Some("written".into()));
731 std::fs::rename(
732 temp.path().join("written/moved.txt"),
733 temp.path().join("before.txt"),
734 )
735 .unwrap();
736 }
737 std::fs::write(temp.path().join("listed/new.txt"), "").unwrap();
739 assert_eq!(changed(&listings), Some("listed".into()));
740 let removed = tempfile::tempdir().unwrap();
742 std::fs::write(removed.path().join("gone.txt"), "").unwrap();
743 std::thread::sleep(std::time::Duration::from_millis(20));
744 let started = now(temp.path());
747 let root = PathSet {
748 listings: [".".to_owned()].into(),
749 ..PathSet::default()
750 };
751 assert_eq!(root.changed_since(removed.path(), started, &[]), None);
752 std::fs::remove_file(removed.path().join("gone.txt")).unwrap();
753 assert_eq!(root.changed_since(removed.path(), started, &[]), Some("."));
754 }
755}