1pub use crate::cc_memo::{CcPreprocessMemo, CcPreprocessMemoInput};
4pub use crate::index_compaction::{IndexCompaction, IndexPageStats, index_page_stats};
5use anyhow::{Context, Result};
6use rusqlite::{Connection, OptionalExtension, Transaction, TransactionBehavior, params};
7use std::path::{Path, PathBuf};
8
9pub const MIN_PERSISTED_HASH_BYTES: i64 = 64 * 1024;
10
11pub const HASH_SETTLE_NS: i64 = 2_000_000_000;
22
23pub fn stamp_is_settled(fingerprint: &FileFingerprint, observed_ns: i64) -> bool {
29 let changed = fingerprint.mtime_ns.max(fingerprint.ctime_ns);
30 observed_ns.saturating_sub(changed) >= HASH_SETTLE_NS
31}
32
33pub const FINE_STAMP_WINDOW_NS: i64 = if cfg!(any(
40 target_os = "linux",
41 target_os = "android",
42 target_vendor = "apple",
43 windows
44)) {
45 1_000_000
46} else {
47 20_000_000
48};
49
50pub const CENTISECOND_STAMP_GRAIN_NS: i64 = 10_000_000;
53
54pub const WHOLE_SECOND_STAMP_GRAIN_NS: i64 = 2_000_000_000;
57
58pub fn stamp_written_since(fingerprint: &FileFingerprint, since_ns: i64) -> bool {
65 [fingerprint.mtime_ns, fingerprint.ctime_ns]
66 .into_iter()
67 .any(|stamp| stamp >= since_ns.saturating_sub(stamp_window_ns(stamp)))
68}
69
70pub fn stamp_window_ns(stamp_ns: i64) -> i64 {
73 let grain = if stamp_ns.rem_euclid(1_000_000_000) == 0 {
74 WHOLE_SECOND_STAMP_GRAIN_NS
75 } else if stamp_ns.rem_euclid(CENTISECOND_STAMP_GRAIN_NS) == 0 {
76 CENTISECOND_STAMP_GRAIN_NS
77 } else {
78 0
79 };
80 grain + FINE_STAMP_WINDOW_NS
81}
82
83#[cfg_attr(windows, allow(unreachable_code))]
89pub fn stamp_clock_ns() -> i64 {
90 #[cfg(any(target_os = "linux", target_os = "android"))]
91 {
92 let mut now: libc::timespec = unsafe { std::mem::zeroed() };
94 if unsafe { libc::clock_gettime(libc::CLOCK_REALTIME_COARSE, &mut now) } == 0 {
95 let elapsed = std::time::Duration::new(now.tv_sec as u64, now.tv_nsec as u32);
96 return i64::try_from(elapsed.as_nanos()).unwrap_or(i64::MAX);
97 }
98 }
99 #[cfg(windows)]
100 {
101 return windows_system_time_ns();
102 }
103 wall_clock_ns()
104}
105
106#[cfg(windows)]
110fn windows_system_time_ns() -> i64 {
111 let mut now = windows_sys::Win32::Foundation::FILETIME {
112 dwLowDateTime: 0,
113 dwHighDateTime: 0,
114 };
115 unsafe { windows_sys::Win32::System::SystemInformation::GetSystemTimeAsFileTime(&mut now) };
117 filetime_to_unix_ns(now.dwHighDateTime, now.dwLowDateTime)
118}
119
120#[cfg(any(windows, test))]
123fn filetime_to_unix_ns(high: u32, low: u32) -> i64 {
124 const UNIX_EPOCH_INTERVALS: u64 = 116_444_736_000_000_000;
125 let intervals = (u64::from(high) << 32) + u64::from(low);
126 intervals
127 .checked_sub(UNIX_EPOCH_INTERVALS)
128 .map_or(0, |since| {
129 i64::try_from(u128::from(since) * 100).unwrap_or(i64::MAX)
130 })
131}
132
133pub fn wall_clock_ns() -> i64 {
135 std::time::SystemTime::now()
136 .duration_since(std::time::UNIX_EPOCH)
137 .map_or(0, |elapsed| {
138 i64::try_from(elapsed.as_nanos()).unwrap_or(i64::MAX)
139 })
140}
141
142#[derive(Debug, Clone, PartialEq, Eq)]
145pub struct ObservedFingerprint {
146 pub fingerprint: FileFingerprint,
147 pub observed_ns: i64,
148}
149
150impl ObservedFingerprint {
151 pub fn from_path(path: &Path) -> Result<Self> {
153 let observed_ns = wall_clock_ns();
154 Ok(Self {
155 fingerprint: FileFingerprint::from_path(path)?,
156 observed_ns,
157 })
158 }
159
160 pub fn settled(&self) -> bool {
162 stamp_is_settled(&self.fingerprint, self.observed_ns)
163 }
164}
165
166const FILE_HASH_LOOKUP_CHUNK: usize = 256;
168
169pub enum FileHashCache<'db> {
170 Borrowed(&'db Connection),
171 #[cfg(any(test, feature = "test-support"))]
172 Owned(Connection),
173}
174
175#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
176pub struct FileFingerprint {
177 pub path: String,
178 pub size: i64,
179 pub mtime_ns: i64,
180 pub ctime_ns: i64,
181 pub inode: i64,
185}
186
187pub enum FileHashLookup {
192 Hit(String),
194 NeedsHash(FileFingerprint),
196 Uncacheable,
198}
199
200impl<'db> FileHashCache<'db> {
201 #[cfg(any(test, feature = "test-support"))]
202 pub fn open(index_db_path: &Path) -> Result<Self> {
203 let db = Connection::open(index_db_path)
204 .with_context(|| format!("opening file hash cache {}", index_db_path.display()))?;
205 db.pragma_update(None, "busy_timeout", "5000")?;
206 db.pragma_update(None, "journal_mode", "WAL")?;
207 db.pragma_update(None, "synchronous", "NORMAL")?;
208 ensure_file_hash_cache_schema(&db)?;
209 Ok(Self::Owned(db))
210 }
211
212 pub fn db(&self) -> &Connection {
213 match self {
214 Self::Borrowed(db) => db,
215 #[cfg(any(test, feature = "test-support"))]
216 Self::Owned(db) => db,
217 }
218 }
219
220 pub fn get(&self, fingerprint: &FileFingerprint) -> rusqlite::Result<Option<String>> {
223 self.db()
224 .query_row(
225 "SELECT hash FROM file_hashes
226 WHERE path = ?1 AND size = ?2 AND mtime_ns = ?3 AND ctime_ns = ?4 AND inode = ?5
227 AND rule >= ?6",
228 params![
229 fingerprint.path,
230 fingerprint.size,
231 fingerprint.mtime_ns,
232 fingerprint.ctime_ns,
233 fingerprint.inode,
234 FILE_HASH_RULE
235 ],
236 |row| row.get(0),
237 )
238 .optional()
239 }
240
241 pub fn get_many(
247 &self,
248 fingerprints: &[&FileFingerprint],
249 ) -> rusqlite::Result<std::collections::HashMap<String, String>> {
250 let mut found = std::collections::HashMap::new();
251 for chunk in fingerprints.chunks(FILE_HASH_LOOKUP_CHUNK) {
252 let wanted: std::collections::HashSet<&FileFingerprint> =
253 chunk.iter().copied().collect();
254 let placeholders = vec!["?"; chunk.len()].join(",");
255 let mut stmt = self.db().prepare_cached(&format!(
256 "SELECT path, size, mtime_ns, ctime_ns, inode, hash FROM file_hashes
257 WHERE path IN ({placeholders}) AND rule >= {FILE_HASH_RULE}"
258 ))?;
259 let rows = stmt.query_map(
260 rusqlite::params_from_iter(chunk.iter().map(|f| f.path.as_str())),
261 |row| {
262 Ok((
263 FileFingerprint {
264 path: row.get(0)?,
265 size: row.get(1)?,
266 mtime_ns: row.get(2)?,
267 ctime_ns: row.get(3)?,
268 inode: row.get(4)?,
269 },
270 row.get::<_, String>(5)?,
271 ))
272 },
273 )?;
274 for row in rows {
275 let (stamp, hash) = row?;
276 if wanted.contains(&stamp) {
277 found.insert(stamp.path, hash);
278 }
279 }
280 }
281 Ok(found)
282 }
283
284 pub fn put(&self, fingerprint: &FileFingerprint, hash: &str) -> rusqlite::Result<()> {
285 self.db().execute(
286 "INSERT OR REPLACE INTO file_hashes
287 (path, size, mtime_ns, ctime_ns, inode, hash, updated_at, rule)
288 VALUES (?1, ?2, ?3, ?4, ?5, ?6, datetime('now'), ?7)",
289 params![
290 fingerprint.path,
291 fingerprint.size,
292 fingerprint.mtime_ns,
293 fingerprint.ctime_ns,
294 fingerprint.inode,
295 hash,
296 FILE_HASH_RULE
297 ],
298 )?;
299 Ok(())
300 }
301
302 pub fn get_source_env_dep_use(
307 &self,
308 content_hash: &str,
309 var: &str,
310 scanner: u32,
311 ) -> rusqlite::Result<Option<i64>> {
312 self.db()
313 .query_row(
314 "SELECT use_kind FROM source_env_dep_uses
315 WHERE content_hash = ?1 AND env_var = ?2 AND scanner = ?3",
316 params![content_hash, var, scanner],
317 |row| row.get(0),
318 )
319 .optional()
320 }
321
322 pub fn put_source_env_dep_use(
323 &self,
324 content_hash: &str,
325 var: &str,
326 scanner: u32,
327 use_kind: i64,
328 ) -> rusqlite::Result<()> {
329 self.db().execute(
330 "INSERT OR REPLACE INTO source_env_dep_uses
331 (content_hash, env_var, scanner, use_kind, updated_at)
332 VALUES (?1, ?2, ?3, ?4, datetime('now'))",
333 params![content_hash, var, scanner, use_kind],
334 )?;
335 Ok(())
336 }
337
338 pub fn has_entry_for_unit(&self, crate_name: &str, unit: &str) -> rusqlite::Result<bool> {
347 if unit.is_empty() {
348 return self.db().query_row(
349 "SELECT EXISTS(SELECT 1 FROM entries WHERE crate_name = ?1)",
350 params![crate_name],
351 |row| row.get(0),
352 );
353 }
354 self.db().query_row(
355 "SELECT EXISTS(SELECT 1 FROM entries
356 WHERE crate_name = ?1 AND unit_id IN (?2, ''))",
357 params![crate_name, unit],
358 |row| row.get(0),
359 )
360 }
361
362 pub fn get_input_prediction(&self, identity: &str) -> rusqlite::Result<Option<(u32, String)>> {
369 self.get_input_prediction_at(identity, unix_now())
370 }
371
372 fn get_input_prediction_at(
373 &self,
374 identity: &str,
375 now: i64,
376 ) -> rusqlite::Result<Option<(u32, String)>> {
377 let row: Option<(u32, String, i64)> = self
378 .db()
379 .query_row(
380 "SELECT schema, prediction_json, last_used FROM input_predictions
381 WHERE identity = ?1",
382 params![identity],
383 |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
384 )
385 .optional()?;
386 let Some((schema, json, last_used)) = row else {
387 return Ok(None);
388 };
389 if prediction_touch_due(last_used, now)
392 && let Err(error) = self.db().execute(
393 "UPDATE input_predictions SET last_used = ?2 WHERE identity = ?1",
394 params![identity, now],
395 )
396 {
397 tracing::debug!("input prediction touch failed: {error}");
398 }
399 Ok(Some((schema, json)))
400 }
401
402 pub fn put_input_prediction(
403 &self,
404 identity: &str,
405 schema: u32,
406 crate_name: Option<&str>,
407 prediction_json: &str,
408 ) -> rusqlite::Result<()> {
409 self.db().execute(
410 "INSERT OR REPLACE INTO input_predictions
411 (identity, schema, crate_name, prediction_json, updated_at, last_used)
412 VALUES (?1, ?2, ?3, ?4, datetime('now'), unixepoch())",
413 params![identity, schema, crate_name, prediction_json],
414 )?;
415 Ok(())
416 }
417
418 pub fn prune_input_predictions(&self) -> rusqlite::Result<usize> {
423 self.prune_input_predictions_at(unix_now())
424 }
425
426 pub fn prune_file_hashes(&self) -> rusqlite::Result<usize> {
432 self.prune_file_hashes_at(unix_now())
433 }
434
435 pub fn rebuild_file_hashes_without_rowid(&self) -> rusqlite::Result<bool> {
444 let tx = Transaction::new_unchecked(self.db(), TransactionBehavior::Immediate)?;
445 if !file_hashes_has_rowid(&tx)? {
446 return Ok(false);
447 }
448 let rule = if file_hashes_have_rule(&tx)? {
449 "rule"
450 } else {
451 "0"
452 };
453 tx.execute_batch(&format!(
454 "CREATE TABLE file_hashes_rebuilt ({FILE_HASHES_COLUMNS}) WITHOUT ROWID;
455 INSERT INTO file_hashes_rebuilt
456 (path, size, mtime_ns, ctime_ns, inode, hash, updated_at, rule)
457 SELECT path, size, mtime_ns, ctime_ns, inode, hash, updated_at, {rule}
458 FROM file_hashes;
459 DROP TABLE file_hashes;
460 ALTER TABLE file_hashes_rebuilt RENAME TO file_hashes;
461 CREATE INDEX IF NOT EXISTS file_hashes_updated_at ON file_hashes(updated_at);"
462 ))?;
463 tx.commit()?;
464 Ok(true)
465 }
466
467 fn prune_file_hashes_at(&self, now: i64) -> rusqlite::Result<usize> {
468 self.db().execute_batch(
471 "CREATE INDEX IF NOT EXISTS file_hashes_updated_at
472 ON file_hashes(updated_at)",
473 )?;
474 self.db().execute(
475 "DELETE FROM file_hashes WHERE updated_at < datetime(?1, 'unixepoch')",
476 params![now.saturating_sub(FILE_HASH_RETENTION_SECS)],
477 )
478 }
479
480 fn prune_input_predictions_at(&self, now: i64) -> rusqlite::Result<usize> {
481 self.db().execute_batch(
484 "CREATE INDEX IF NOT EXISTS input_predictions_last_used
485 ON input_predictions(last_used)",
486 )?;
487 self.db().execute(
488 "DELETE FROM input_predictions WHERE last_used < ?1",
489 params![prediction_prune_cutoff(now)],
490 )
491 }
492}
493
494pub const FILE_HASH_RETENTION_SECS: i64 = 30 * 86_400;
500
501pub const INPUT_PREDICTION_TOUCH_INTERVAL_SECS: i64 = 86_400;
503
504pub const INPUT_PREDICTION_RETENTION_SECS: i64 = 30 * 86_400;
508
509fn unix_now() -> i64 {
510 std::time::SystemTime::now()
511 .duration_since(std::time::UNIX_EPOCH)
512 .map_or(0, |elapsed| elapsed.as_secs() as i64)
513}
514
515fn prediction_touch_due(last_used: i64, now: i64) -> bool {
516 now.saturating_sub(last_used) >= INPUT_PREDICTION_TOUCH_INTERVAL_SECS
517}
518
519fn prediction_prune_cutoff(now: i64) -> i64 {
520 now.saturating_sub(INPUT_PREDICTION_RETENTION_SECS)
521}
522
523fn input_predictions_have_last_used(db: &Connection) -> rusqlite::Result<bool> {
524 db.query_row(
525 "SELECT EXISTS(SELECT 1 FROM pragma_table_info('input_predictions') WHERE name = 'last_used')",
526 [],
527 |row| row.get(0),
528 )
529}
530
531fn ensure_input_predictions_last_used(db: &Connection, now: i64) -> rusqlite::Result<()> {
542 if input_predictions_have_last_used(db)? {
543 return Ok(());
544 }
545 let tx = Transaction::new_unchecked(db, TransactionBehavior::Immediate)?;
548 if !input_predictions_have_last_used(&tx)? {
549 tx.execute_batch(&format!(
550 "ALTER TABLE input_predictions ADD COLUMN last_used INTEGER NOT NULL DEFAULT {now}"
551 ))?;
552 }
553 tx.commit()
554}
555
556const FILE_HASHES_COLUMNS: &str = "
560 path TEXT PRIMARY KEY,
561 size INTEGER NOT NULL,
562 mtime_ns INTEGER NOT NULL,
563 ctime_ns INTEGER NOT NULL DEFAULT 0,
564 inode INTEGER NOT NULL DEFAULT 0,
565 hash TEXT NOT NULL,
566 updated_at TEXT NOT NULL DEFAULT (datetime('now')),
567 rule INTEGER NOT NULL DEFAULT 0";
568
569pub const FILE_HASH_RULE: i64 = 1;
577
578fn file_hashes_have_rule(db: &Connection) -> rusqlite::Result<bool> {
579 db.query_row(
580 "SELECT EXISTS(SELECT 1 FROM pragma_table_info('file_hashes') WHERE name = 'rule')",
581 [],
582 |row| row.get(0),
583 )
584}
585
586fn ensure_file_hashes_rule(db: &Connection) -> rusqlite::Result<()> {
596 if file_hashes_have_rule(db)? {
597 return Ok(());
598 }
599 let tx = Transaction::new_unchecked(db, TransactionBehavior::Immediate)?;
600 if !file_hashes_have_rule(&tx)? {
601 tx.execute_batch(
602 "ALTER TABLE file_hashes ADD COLUMN rule INTEGER NOT NULL DEFAULT 0;
603 DELETE FROM file_hashes;",
604 )?;
605 }
606 tx.commit()
607}
608
609fn file_hashes_has_rowid(db: &Connection) -> rusqlite::Result<bool> {
613 let sql: Option<String> = db
614 .query_row(
615 "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'file_hashes'",
616 [],
617 |row| row.get(0),
618 )
619 .optional()?;
620 Ok(sql.is_some_and(|sql| !sql.to_ascii_uppercase().contains("WITHOUT ROWID")))
621}
622
623pub(crate) fn has_rule_columns(db: &Connection) -> rusqlite::Result<bool> {
632 Ok(file_hashes_have_rule(db)? && crate::cc_memo::has_rule_columns(db)?)
633}
634
635pub fn ensure_file_hash_cache_schema(db: &Connection) -> rusqlite::Result<()> {
636 crate::cc_memo::ensure_schema(db)?;
637 crate::cc_memo::ensure_mapped_hash_schema(db)?;
638 db.execute_batch(&format!(
641 "CREATE TABLE IF NOT EXISTS file_hashes ({FILE_HASHES_COLUMNS}) WITHOUT ROWID;"
642 ))?;
643 db.execute_batch(
644 "CREATE TABLE IF NOT EXISTS input_predictions (
645 identity TEXT PRIMARY KEY,
646 schema INTEGER NOT NULL,
647 crate_name TEXT,
648 prediction_json TEXT NOT NULL,
649 updated_at TEXT NOT NULL DEFAULT (datetime('now')),
650 last_used INTEGER NOT NULL DEFAULT (unixepoch())
651 );
652 CREATE TABLE IF NOT EXISTS source_env_dep_uses (
653 content_hash TEXT NOT NULL,
654 env_var TEXT NOT NULL,
655 scanner INTEGER NOT NULL,
656 use_kind INTEGER NOT NULL,
657 updated_at TEXT NOT NULL DEFAULT (datetime('now')),
658 PRIMARY KEY (content_hash, env_var)
659 );
660 -- Boolean answers from the first env-use scanner, which missed
661 -- computed names and other spellings. Nothing reads them any more.
662 DROP TABLE IF EXISTS source_env_runtime_uses;",
663 )?;
664 for column in [
665 "ALTER TABLE file_hashes ADD COLUMN ctime_ns INTEGER NOT NULL DEFAULT 0",
666 "ALTER TABLE file_hashes ADD COLUMN inode INTEGER NOT NULL DEFAULT 0",
667 ] {
668 if let Err(e) = db.execute_batch(column)
669 && !e.to_string().contains("duplicate column name")
670 {
671 return Err(e);
672 }
673 }
674 ensure_file_hashes_rule(db)?;
675 crate::cc_memo::ensure_rule(db)?;
676 ensure_input_predictions_last_used(db, unix_now())
677}
678
679impl FileFingerprint {
680 pub fn from_path(path: &Path) -> Result<Self> {
686 let metadata = std::fs::metadata(path)
687 .with_context(|| format!("reading metadata for {}", path.display()))?;
688 let absolute_path = absolute_path(path);
689
690 Ok(Self {
691 path: absolute_path.to_string_lossy().into_owned(),
692 size: i64::try_from(metadata.len()).unwrap_or(i64::MAX),
693 mtime_ns: metadata_mtime_ns(&metadata),
694 ctime_ns: metadata_ctime_ns(&metadata),
695 inode: metadata_inode(&metadata),
696 })
697 }
698}
699
700pub fn absolute_path(path: &Path) -> PathBuf {
701 if path.is_absolute() {
702 path.to_path_buf()
703 } else {
704 std::env::current_dir()
705 .map(|cwd| cwd.join(path))
706 .unwrap_or_else(|_| path.to_path_buf())
707 }
708}
709
710pub fn metadata_inode(metadata: &std::fs::Metadata) -> i64 {
712 #[cfg(unix)]
713 {
714 use std::os::unix::fs::MetadataExt;
715 i64::try_from(metadata.ino()).unwrap_or(i64::MAX)
716 }
717 #[cfg(not(unix))]
718 {
719 let _ = metadata;
720 0
721 }
722}
723
724pub fn metadata_mtime_ns(metadata: &std::fs::Metadata) -> i64 {
725 #[cfg(unix)]
726 {
727 use std::os::unix::fs::MetadataExt;
728
729 metadata_parts_ns(metadata.mtime(), metadata.mtime_nsec())
730 }
731
732 #[cfg(windows)]
733 {
734 use std::os::windows::fs::MetadataExt;
735
736 windows_filetime_ns(metadata.last_write_time())
737 }
738
739 #[cfg(not(any(unix, windows)))]
740 {
741 system_time_ns(metadata.modified().ok()).unwrap_or_default()
742 }
743}
744
745pub fn metadata_ctime_ns(metadata: &std::fs::Metadata) -> i64 {
746 #[cfg(unix)]
747 {
748 use std::os::unix::fs::MetadataExt;
749
750 metadata_parts_ns(metadata.ctime(), metadata.ctime_nsec())
751 }
752
753 #[cfg(windows)]
754 {
755 use std::os::windows::fs::MetadataExt;
756
757 windows_filetime_ns(metadata.creation_time())
758 }
759
760 #[cfg(not(any(unix, windows)))]
761 {
762 system_time_ns(metadata.created().ok()).unwrap_or_else(|| metadata_mtime_ns(metadata))
763 }
764}
765
766#[cfg(unix)]
767fn metadata_parts_ns(seconds: i64, nanoseconds: i64) -> i64 {
768 seconds
769 .saturating_mul(1_000_000_000)
770 .saturating_add(nanoseconds)
771}
772
773#[cfg(any(windows, test))]
774fn windows_filetime_ns(filetime_100ns: u64) -> i64 {
775 const UNIX_EPOCH_FILETIME_100NS: u64 = 116_444_736_000_000_000;
776
777 filetime_100ns
778 .saturating_sub(UNIX_EPOCH_FILETIME_100NS)
779 .saturating_mul(100)
780 .min(i64::MAX as u64) as i64
781}
782
783#[cfg(any(test, not(any(unix, windows))))]
784fn system_time_ns(time: Option<std::time::SystemTime>) -> Option<i64> {
785 let duration = time?.duration_since(std::time::UNIX_EPOCH).ok()?;
786 Some(i64::try_from(duration.as_nanos()).unwrap_or(i64::MAX))
787}
788
789pub fn hash_file(path: &Path) -> Result<String> {
791 let file = std::fs::File::open(path)
792 .with_context(|| format!("opening {} for hashing", path.display()))?;
793 let mut hasher = blake3::Hasher::new();
794 hasher
795 .update_reader(file)
796 .with_context(|| format!("reading {} for hashing", path.display()))?;
797 Ok(hasher.finalize().to_hex().to_string())
798}
799
800impl FileHashCache<'_> {
801 pub fn lookup_cached(&self, path: &Path) -> FileHashLookup {
807 let cache = self;
808 let fingerprint = match FileFingerprint::from_path(path) {
809 Ok(fp) => fp,
810 Err(e) => {
811 tracing::debug!(
812 "file hash cache metadata lookup failed for {}: {e}",
813 path.display()
814 );
815 return FileHashLookup::Uncacheable;
816 }
817 };
818 if fingerprint.size < MIN_PERSISTED_HASH_BYTES {
819 return FileHashLookup::Uncacheable;
820 }
821 match cache.get(&fingerprint) {
822 Ok(Some(hash)) => FileHashLookup::Hit(hash),
823 Ok(None) => FileHashLookup::NeedsHash(fingerprint),
824 Err(e) => {
825 tracing::debug!("file hash cache lookup failed for {}: {e}", path.display());
827 FileHashLookup::NeedsHash(fingerprint)
828 }
829 }
830 }
831 pub fn record_cached(&self, fingerprint: &FileFingerprint, hash: &str) {
835 if let Err(e) = self.put(fingerprint, hash) {
836 tracing::debug!("file hash cache update failed: {e}");
837 }
838 }
839 pub fn record_verified(&self, fingerprint: &FileFingerprint, hash: &str) {
853 if fingerprint.size < MIN_PERSISTED_HASH_BYTES {
854 return;
855 }
856 self.record_cached(fingerprint, hash);
857 }
858}
859
860#[cfg(test)]
861mod tests {
862 use super::*;
863
864 const ROWID_FILE_HASHES: &str = "CREATE TABLE file_hashes (
866 path TEXT PRIMARY KEY,
867 size INTEGER NOT NULL,
868 mtime_ns INTEGER NOT NULL,
869 ctime_ns INTEGER NOT NULL DEFAULT 0,
870 inode INTEGER NOT NULL DEFAULT 0,
871 hash TEXT NOT NULL,
872 updated_at TEXT NOT NULL DEFAULT (datetime('now'))
873 )";
874
875 #[test]
876 fn a_new_store_keeps_file_hashes_without_rowid() {
877 let db = rusqlite::Connection::open_in_memory().unwrap();
878 ensure_file_hash_cache_schema(&db).unwrap();
879 assert!(!file_hashes_has_rowid(&db).unwrap());
880 let cache = FileHashCache::Borrowed(&db);
881 assert!(!cache.rebuild_file_hashes_without_rowid().unwrap());
882 }
883
884 #[test]
885 fn an_older_file_hashes_table_is_rebuilt_once_with_its_rows() {
886 let db = rusqlite::Connection::open_in_memory().unwrap();
887 db.execute_batch(ROWID_FILE_HASHES).unwrap();
888 ensure_file_hash_cache_schema(&db).unwrap();
889 assert!(
890 file_hashes_has_rowid(&db).unwrap(),
891 "opening does not rebuild"
892 );
893 db.execute(
894 "INSERT INTO file_hashes (path, size, mtime_ns, ctime_ns, inode, hash, updated_at)
895 VALUES ('/a', 1, 2, 3, 4, 'h', '2026-09-01 00:00:00')",
896 [],
897 )
898 .unwrap();
899
900 let cache = FileHashCache::Borrowed(&db);
901 assert!(cache.rebuild_file_hashes_without_rowid().unwrap());
902 assert!(!file_hashes_has_rowid(&db).unwrap());
903 let row: (String, i64, i64, i64, i64, String, String) = db
904 .query_row(
905 "SELECT path, size, mtime_ns, ctime_ns, inode, hash, updated_at FROM file_hashes",
906 [],
907 |r| {
908 Ok((
909 r.get(0)?,
910 r.get(1)?,
911 r.get(2)?,
912 r.get(3)?,
913 r.get(4)?,
914 r.get(5)?,
915 r.get(6)?,
916 ))
917 },
918 )
919 .unwrap();
920 assert_eq!(
921 row,
922 (
923 "/a".into(),
924 1,
925 2,
926 3,
927 4,
928 "h".into(),
929 "2026-09-01 00:00:00".into()
930 )
931 );
932 let indexed: bool = db
933 .query_row(
934 "SELECT EXISTS(SELECT 1 FROM sqlite_master
935 WHERE type = 'index' AND name = 'file_hashes_updated_at')",
936 [],
937 |r| r.get(0),
938 )
939 .unwrap();
940 assert!(indexed, "the prune's index comes back");
941 assert!(!cache.rebuild_file_hashes_without_rowid().unwrap(), "once");
942 }
943
944 #[test]
945 fn the_file_hash_window_is_thirty_days() {
946 assert_eq!(FILE_HASH_RETENTION_SECS, 2_592_000);
947 }
948
949 #[test]
950 fn file_hash_rows_are_pruned_a_month_after_they_were_written() {
951 let db = rusqlite::Connection::open_in_memory().unwrap();
952 ensure_file_hash_cache_schema(&db).unwrap();
953 let cache = FileHashCache::Borrowed(&db);
954 let now = 1_700_000_000_i64;
956 for (path, age) in [
957 ("/kept", FILE_HASH_RETENTION_SECS - 60),
958 ("/boundary", FILE_HASH_RETENTION_SECS),
959 ("/old", FILE_HASH_RETENTION_SECS + 60),
960 ] {
961 db.execute(
962 "INSERT INTO file_hashes (path, size, mtime_ns, hash, updated_at)
963 VALUES (?1, 1, 1, 'h', datetime(?2, 'unixepoch'))",
964 rusqlite::params![path, now - age],
965 )
966 .unwrap();
967 }
968 assert_eq!(cache.prune_file_hashes_at(now).unwrap(), 1);
969 let mut left: Vec<String> = db
970 .prepare("SELECT path FROM file_hashes ORDER BY path")
971 .unwrap()
972 .query_map([], |row| row.get(0))
973 .unwrap()
974 .map(Result::unwrap)
975 .collect();
976 left.sort();
977 assert_eq!(
978 left,
979 ["/boundary", "/kept"],
980 "a row exactly a month old stays"
981 );
982 assert_eq!(
984 cache.prune_file_hashes().unwrap(),
985 2,
986 "real clock: both are years old"
987 );
988 }
989
990 fn stamp(path: &str, mtime_ns: i64, ctime_ns: i64) -> FileFingerprint {
991 FileFingerprint {
992 path: path.to_string(),
993 size: 10,
994 mtime_ns,
995 ctime_ns,
996 inode: 7,
997 }
998 }
999
1000 #[test]
1001 fn a_stamp_settles_exactly_one_window_after_its_last_change() {
1002 let changed = 1_000_000_000_000;
1003 let file = stamp("/h.h", changed, changed);
1004 assert!(!stamp_is_settled(&file, changed));
1005 assert!(!stamp_is_settled(&file, changed + HASH_SETTLE_NS - 1));
1006 assert!(stamp_is_settled(&file, changed + HASH_SETTLE_NS));
1007 assert!(!stamp_is_settled(&file, changed - 1));
1009 }
1010
1011 #[test]
1012 fn a_write_counts_from_one_stamp_window_before_the_start() {
1013 let fine = if cfg!(any(
1014 target_os = "linux",
1015 target_os = "android",
1016 target_vendor = "apple",
1017 windows
1018 )) {
1019 1_000_000
1020 } else {
1021 20_000_000
1022 };
1023 assert_eq!(FINE_STAMP_WINDOW_NS, fine);
1024 let both = |ns| stamp("/s.rs", ns, ns);
1025 let whole = 1_700_000_000_000_000_000;
1026 let whole_window = 2_000_000_000 + fine;
1027 assert!(stamp_written_since(&both(whole), whole + whole_window));
1028 assert!(!stamp_written_since(&both(whole), whole + whole_window + 1));
1029 let centi = whole + 120_000_000;
1030 let centi_window = 10_000_000 + fine;
1031 assert!(stamp_written_since(&both(centi), centi + centi_window));
1032 assert!(!stamp_written_since(&both(centi), centi + centi_window + 1));
1033 let precise = whole + 123_456_789;
1034 assert!(stamp_written_since(&both(precise), precise + fine));
1035 assert!(!stamp_written_since(&both(precise), precise + fine + 1));
1036 let long_ago = whole - 60_000_000_000;
1040 let fat = stamp("/s.rs", whole, long_ago + 30_000_000);
1041 assert!(stamp_written_since(&fat, whole + whole_window));
1042 assert!(!stamp_written_since(&fat, whole + whole_window + 1));
1043 let restored = stamp("/s.rs", long_ago, precise);
1044 assert!(stamp_written_since(&restored, precise + fine));
1045 assert!(!stamp_written_since(&restored, precise + fine + 1));
1046 }
1047
1048 #[test]
1049 fn a_filetime_counts_from_the_unix_epoch_in_nanoseconds() {
1050 let split = |intervals: u64| ((intervals >> 32) as u32, intervals as u32);
1051 let epoch = 116_444_736_000_000_000_u64;
1052 let at = |intervals: u64| {
1053 let (high, low) = split(intervals);
1054 filetime_to_unix_ns(high, low)
1055 };
1056 assert_eq!(at(epoch), 0);
1057 assert_eq!(at(epoch + 1), 100);
1058 assert_eq!(at(epoch - 1), 0);
1059 assert_eq!(
1061 at(epoch + 1_791_590_400 * 10_000_000),
1062 1_791_590_400_000_000_000
1063 );
1064 assert_eq!(filetime_to_unix_ns(u32::MAX, u32::MAX), i64::MAX);
1065 }
1066
1067 #[cfg(windows)]
1071 #[test]
1072 fn a_file_written_after_the_stamp_clock_counts_as_written_since() {
1073 let dir = tempfile::tempdir().unwrap();
1074 let path = dir.path().join("written.rs");
1075 for round in 0..200 {
1076 let start = stamp_clock_ns();
1077 std::fs::write(&path, round.to_string()).unwrap();
1078 let written = FileFingerprint::from_path(&path).unwrap();
1079 assert!(
1080 stamp_written_since(&written, start),
1081 "round {round}: {written:?} before {start}"
1082 );
1083 }
1084 }
1085
1086 #[test]
1090 fn a_write_right_after_the_stamp_clock_counts_as_written_since() {
1091 let dir = tempfile::tempdir().unwrap();
1092 let file = dir.path().join("f");
1093 for round in 0..500 {
1094 let since = stamp_clock_ns();
1095 std::fs::write(&file, round.to_string()).unwrap();
1096 let written = FileFingerprint::from_path(&file).unwrap();
1097 assert!(
1098 stamp_written_since(&written, since),
1099 "round {round}: {written:?} since {since}"
1100 );
1101 }
1102 }
1103
1104 #[test]
1107 fn a_write_stops_counting_once_the_stamp_clock_passes_its_window() {
1108 let dir = tempfile::tempdir().unwrap();
1109 let file = dir.path().join("f");
1110 std::fs::write(&file, "x").unwrap();
1111 let written = FileFingerprint::from_path(&file).unwrap();
1112 let give_up = std::time::Instant::now() + std::time::Duration::from_secs(10);
1114 while stamp_written_since(&written, stamp_clock_ns()) {
1115 assert!(
1116 std::time::Instant::now() < give_up,
1117 "the stamp clock never passed {written:?}: {}",
1118 stamp_clock_ns()
1119 );
1120 std::thread::sleep(std::time::Duration::from_millis(1));
1121 }
1122 }
1123
1124 #[test]
1125 fn a_recent_ctime_holds_back_a_file_whose_mtime_was_restored() {
1126 let now = 1_000_000_000_000;
1129 let old = now - 10 * HASH_SETTLE_NS;
1130 assert!(!stamp_is_settled(&stamp("/h.h", old, now - 1), now));
1131 assert!(stamp_is_settled(&stamp("/h.h", old, old), now));
1132 }
1133
1134 #[test]
1137 fn an_observation_carries_the_wall_clock_it_was_taken_at() {
1138 let since_epoch = || {
1139 let elapsed = std::time::SystemTime::now()
1140 .duration_since(std::time::UNIX_EPOCH)
1141 .unwrap();
1142 i64::try_from(elapsed.as_nanos()).unwrap()
1143 };
1144 let dir = tempfile::tempdir().unwrap();
1145 let file = dir.path().join("f");
1146 std::fs::write(&file, "x").unwrap();
1147 let before = since_epoch();
1148 let observed = ObservedFingerprint::from_path(&file).unwrap();
1149 let after = since_epoch();
1150 assert!(
1151 (before..=after).contains(&observed.observed_ns),
1152 "{observed:?} outside {before}..={after}"
1153 );
1154 assert_eq!(
1155 observed.fingerprint,
1156 FileFingerprint::from_path(&file).unwrap()
1157 );
1158 }
1159
1160 #[test]
1163 fn an_observation_is_settled_by_the_time_it_was_taken() {
1164 let changed = 1_000_000_000_000;
1165 let observed_at = |observed_ns| ObservedFingerprint {
1166 fingerprint: stamp("/h.h", changed, changed),
1167 observed_ns,
1168 };
1169 assert!(!observed_at(changed + HASH_SETTLE_NS - 1).settled());
1170 assert!(observed_at(changed + HASH_SETTLE_NS).settled());
1171 }
1172
1173 #[test]
1174 fn many_lookups_return_only_rows_whose_whole_stamp_matches() {
1175 let db = Connection::open_in_memory().unwrap();
1176 ensure_file_hash_cache_schema(&db).unwrap();
1177 let cache = FileHashCache::Borrowed(&db);
1178 let same = stamp("/same.h", 1, 1);
1179 let moved = stamp("/moved.h", 1, 1);
1180 cache.put(&same, "h-same").unwrap();
1181 cache.put(&moved, "h-moved").unwrap();
1182 let mut rewritten = moved.clone();
1183 rewritten.mtime_ns = 2;
1184 let absent = stamp("/absent.h", 1, 1);
1185
1186 let found = cache.get_many(&[&same, &rewritten, &absent]).unwrap();
1187 assert_eq!(found.len(), 1, "{found:?}");
1188 assert_eq!(found["/same.h"], "h-same");
1189 assert!(
1190 !found.contains_key("/moved.h"),
1191 "a changed stamp reads as absent"
1192 );
1193 }
1194
1195 fn record_as_older_release(db: &Connection, fingerprint: &FileFingerprint, hash: &str) {
1197 db.execute(
1198 "INSERT OR REPLACE INTO file_hashes
1199 (path, size, mtime_ns, ctime_ns, inode, hash, updated_at)
1200 VALUES (?1, ?2, ?3, ?4, ?5, ?6, datetime('now'))",
1201 params![
1202 fingerprint.path,
1203 fingerprint.size,
1204 fingerprint.mtime_ns,
1205 fingerprint.ctime_ns,
1206 fingerprint.inode,
1207 hash
1208 ],
1209 )
1210 .unwrap();
1211 }
1212
1213 #[test]
1214 fn a_row_an_older_release_records_is_never_served() {
1215 let db = Connection::open_in_memory().unwrap();
1216 ensure_file_hash_cache_schema(&db).unwrap();
1217 let cache = FileHashCache::Borrowed(&db);
1218 let current = stamp("/current.h", 1, 1);
1219 let older = stamp("/older.h", 1, 1);
1220 cache.put(¤t, "h-current").unwrap();
1221 record_as_older_release(&db, &older, "h-older");
1222
1223 assert_eq!(cache.get(¤t).unwrap().as_deref(), Some("h-current"));
1224 assert_eq!(cache.get(&older).unwrap(), None);
1225 let found = cache.get_many(&[¤t, &older]).unwrap();
1226 assert_eq!(found.len(), 1, "{found:?}");
1227 assert_eq!(found["/current.h"], "h-current");
1228
1229 record_as_older_release(&db, ¤t, "h-older");
1231 assert_eq!(cache.get(¤t).unwrap(), None);
1232 }
1233
1234 #[test]
1235 fn the_rule_column_arrives_once_and_takes_the_older_rows_with_it() {
1236 for table in [
1237 ROWID_FILE_HASHES.to_string(),
1238 format!("{ROWID_FILE_HASHES} WITHOUT ROWID"),
1239 ] {
1240 let db = Connection::open_in_memory().unwrap();
1241 db.execute_batch(&table).unwrap();
1242 record_as_older_release(&db, &stamp("/older.h", 1, 1), "h-older");
1243 ensure_file_hash_cache_schema(&db).unwrap();
1244 assert!(file_hashes_have_rule(&db).unwrap(), "{table}");
1245 let rows = || -> i64 {
1246 db.query_row("SELECT count(*) FROM file_hashes", [], |row| row.get(0))
1247 .unwrap()
1248 };
1249 assert_eq!(rows(), 0, "{table}");
1250
1251 let cache = FileHashCache::Borrowed(&db);
1252 cache.put(&stamp("/current.h", 1, 1), "h-current").unwrap();
1253 ensure_file_hash_cache_schema(&db).unwrap();
1254 assert_eq!(rows(), 1, "{table}: once");
1255 }
1256 }
1257
1258 #[test]
1259 fn rebuilding_without_rowid_keeps_each_rows_rule() {
1260 let db = Connection::open_in_memory().unwrap();
1261 db.execute_batch(ROWID_FILE_HASHES).unwrap();
1262 ensure_file_hash_cache_schema(&db).unwrap();
1263 let cache = FileHashCache::Borrowed(&db);
1264 let current = stamp("/current.h", 1, 1);
1265 let older = stamp("/older.h", 1, 1);
1266 cache.put(¤t, "h-current").unwrap();
1267 record_as_older_release(&db, &older, "h-older");
1268
1269 assert!(cache.rebuild_file_hashes_without_rowid().unwrap());
1270 assert_eq!(cache.get(¤t).unwrap().as_deref(), Some("h-current"));
1271 assert_eq!(cache.get(&older).unwrap(), None);
1272 }
1273
1274 #[test]
1275 fn many_lookups_span_more_paths_than_one_statement_binds() {
1276 let db = Connection::open_in_memory().unwrap();
1277 ensure_file_hash_cache_schema(&db).unwrap();
1278 let cache = FileHashCache::Borrowed(&db);
1279 let stamps: Vec<FileFingerprint> = (0..FILE_HASH_LOOKUP_CHUNK * 2 + 3)
1280 .map(|i| stamp(&format!("/h{i}.h"), 1, 1))
1281 .collect();
1282 for (i, s) in stamps.iter().enumerate() {
1283 cache.put(s, &format!("hash{i}")).unwrap();
1284 }
1285 let refs: Vec<&FileFingerprint> = stamps.iter().collect();
1286 let found = cache.get_many(&refs).unwrap();
1287 assert_eq!(found.len(), stamps.len());
1288 assert_eq!(found["/h0.h"], "hash0");
1289 let last = stamps.len() - 1;
1290 assert_eq!(found[&format!("/h{last}.h")], format!("hash{last}"));
1291 }
1292
1293 #[test]
1294 fn has_entry_for_unit_sees_only_that_crate_and_unit() {
1295 let db = Connection::open_in_memory().unwrap();
1296 db.execute_batch(
1297 "CREATE TABLE entries (cache_key TEXT PRIMARY KEY, crate_name TEXT NOT NULL,
1298 unit_id TEXT NOT NULL DEFAULT '');",
1299 )
1300 .unwrap();
1301 let cache = FileHashCache::Borrowed(&db);
1302 assert!(!cache.has_entry_for_unit("gpui", "").unwrap());
1303 assert!(!cache.has_entry_for_unit("gpui", "u1").unwrap());
1304 db.execute(
1305 "INSERT INTO entries (cache_key, crate_name, unit_id) VALUES ('k', 'gpui_base', 'u1')",
1306 [],
1307 )
1308 .unwrap();
1309 assert!(!cache.has_entry_for_unit("gpui", "u1").unwrap());
1310 assert!(cache.has_entry_for_unit("gpui_base", "u1").unwrap());
1311 assert!(
1312 cache.has_entry_for_unit("gpui_base", "").unwrap(),
1313 "any unit"
1314 );
1315 assert!(
1316 !cache.has_entry_for_unit("gpui_base", "u2").unwrap(),
1317 "another unit of the same crate name is absent"
1318 );
1319 db.execute(
1320 "INSERT INTO entries (cache_key, crate_name) VALUES ('k2', 'build_script_build')",
1321 [],
1322 )
1323 .unwrap();
1324 assert!(
1325 cache
1326 .has_entry_for_unit("build_script_build", "u9")
1327 .unwrap(),
1328 "a row with no unit stands for every unit of its name"
1329 );
1330 }
1331
1332 #[test]
1333 fn fingerprint_preserves_the_filesystem_identity() {
1334 let dir = tempfile::tempdir().unwrap();
1335 let file = dir.path().join("artifact.rlib");
1336 std::fs::write(&file, vec![7; 65_537]).unwrap();
1337 #[cfg(any(unix, windows))]
1338 let metadata = std::fs::metadata(&file).unwrap();
1339 let fingerprint = FileFingerprint::from_path(&file).unwrap();
1340 assert_eq!(fingerprint.path, file.to_string_lossy());
1341 assert_eq!(fingerprint.size, 65_537);
1342
1343 #[cfg(unix)]
1344 {
1345 use std::os::unix::fs::MetadataExt;
1346 assert_eq!(fingerprint.inode, metadata.ino() as i64);
1347 assert_eq!(
1348 fingerprint.mtime_ns,
1349 metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec()
1350 );
1351 assert_eq!(
1352 fingerprint.ctime_ns,
1353 metadata.ctime() * 1_000_000_000 + metadata.ctime_nsec()
1354 );
1355 }
1356 #[cfg(windows)]
1357 {
1358 use std::os::windows::fs::MetadataExt;
1359 assert_eq!(fingerprint.inode, 0);
1360 assert_eq!(
1361 fingerprint.mtime_ns,
1362 ((metadata.last_write_time() - 116_444_736_000_000_000) * 100) as i64
1363 );
1364 assert_eq!(
1365 fingerprint.ctime_ns,
1366 ((metadata.creation_time() - 116_444_736_000_000_000) * 100) as i64
1367 );
1368 }
1369 assert_eq!(
1370 absolute_path(Path::new("relative.rlib")),
1371 std::env::current_dir().unwrap().join("relative.rlib")
1372 );
1373 }
1374
1375 #[test]
1376 fn windows_filetime_converts_epoch_units_and_saturates() {
1377 assert_eq!(windows_filetime_ns(0), 0);
1378 assert_eq!(windows_filetime_ns(116_444_735_999_999_999), 0);
1379 assert_eq!(windows_filetime_ns(116_444_736_000_000_000), 0);
1380 assert_eq!(windows_filetime_ns(116_444_736_000_000_001), 100);
1381 assert_eq!(windows_filetime_ns(116_444_736_012_345_678), 1_234_567_800);
1382 assert_eq!(windows_filetime_ns(u64::MAX), i64::MAX);
1383 }
1384
1385 #[test]
1386 fn system_time_conversion_rejects_missing_or_pre_epoch_values() {
1387 use std::time::{Duration, UNIX_EPOCH};
1388
1389 assert_eq!(system_time_ns(None), None);
1391 assert_eq!(
1392 system_time_ns(Some(UNIX_EPOCH - Duration::from_nanos(100))),
1393 None
1394 );
1395 assert_eq!(system_time_ns(Some(UNIX_EPOCH)), Some(0));
1396 assert_eq!(
1397 system_time_ns(Some(UNIX_EPOCH + Duration::new(1, 234_567_800))),
1398 Some(1_234_567_800)
1399 );
1400 assert_eq!(
1401 system_time_ns(Some(UNIX_EPOCH + Duration::from_secs(9_223_372_037))),
1402 Some(i64::MAX)
1403 );
1404 }
1405
1406 #[test]
1407 fn env_dep_use_memo_persists_each_answer_per_content_variable_and_scanner() {
1408 let dir = tempfile::tempdir().unwrap();
1409 let db_path = dir.path().join("index.db");
1410 let cache = FileHashCache::open(&db_path).unwrap();
1411 assert_eq!(
1412 cache
1413 .get_source_env_dep_use("source-a", "OUT_DIR", 2)
1414 .unwrap(),
1415 None
1416 );
1417 cache
1418 .put_source_env_dep_use("source-a", "OUT_DIR", 2, 1)
1419 .unwrap();
1420 cache
1421 .put_source_env_dep_use("source-a", "OTHER", 2, 0)
1422 .unwrap();
1423 cache
1424 .put_source_env_dep_use("source-b", "OUT_DIR", 2, 2)
1425 .unwrap();
1426 drop(cache);
1427
1428 let cache = FileHashCache::open(&db_path).unwrap();
1429 let get = |content: &str, var: &str, scanner: u32| {
1430 cache.get_source_env_dep_use(content, var, scanner).unwrap()
1431 };
1432 assert_eq!(get("source-a", "OUT_DIR", 2), Some(1));
1433 assert_eq!(get("source-a", "OTHER", 2), Some(0));
1434 assert_eq!(get("source-b", "OUT_DIR", 2), Some(2));
1435 assert_eq!(get("source-b", "OTHER", 2), None);
1436 assert_eq!(get("source-a", "OUT_DIR", 1), None);
1438 assert_eq!(get("source-a", "OUT_DIR", 3), None);
1439
1440 cache
1442 .put_source_env_dep_use("source-a", "OUT_DIR", 3, 2)
1443 .unwrap();
1444 assert_eq!(get("source-a", "OUT_DIR", 3), Some(2));
1445 assert_eq!(get("source-a", "OUT_DIR", 2), None);
1446 }
1447
1448 #[test]
1449 fn schema_drops_the_boolean_env_use_memo() {
1450 let conn = rusqlite::Connection::open_in_memory().unwrap();
1451 conn.execute_batch(
1452 "CREATE TABLE source_env_runtime_uses (
1453 content_hash TEXT NOT NULL,
1454 env_var TEXT NOT NULL,
1455 has_runtime_use INTEGER NOT NULL,
1456 updated_at TEXT NOT NULL DEFAULT (datetime('now')),
1457 PRIMARY KEY (content_hash, env_var)
1458 );
1459 INSERT INTO source_env_runtime_uses (content_hash, env_var, has_runtime_use)
1460 VALUES ('source-a', 'OUT_DIR', 0);",
1461 )
1462 .unwrap();
1463 ensure_file_hash_cache_schema(&conn).unwrap();
1464 let tables = |name: &str| -> i64 {
1465 conn.query_row(
1466 "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = ?1",
1467 params![name],
1468 |row| row.get(0),
1469 )
1470 .unwrap()
1471 };
1472 assert_eq!(tables("source_env_runtime_uses"), 0);
1473 assert_eq!(tables("source_env_dep_uses"), 1);
1474 }
1475
1476 #[test]
1477 fn input_prediction_preserves_the_callers_schema_and_opaque_payload() {
1478 let dir = tempfile::tempdir().unwrap();
1479 let db_path = dir.path().join("index.db");
1480 let cache = FileHashCache::open(&db_path).unwrap();
1481 assert_eq!(cache.get_input_prediction("first").unwrap(), None);
1482 cache
1483 .put_input_prediction("first", 17, Some("example"), "payload-a")
1484 .unwrap();
1485 cache
1486 .put_input_prediction("second", 42, None, "payload-b")
1487 .unwrap();
1488 drop(cache);
1489
1490 let cache = FileHashCache::open(&db_path).unwrap();
1491 assert_eq!(
1492 cache.get_input_prediction("first").unwrap(),
1493 Some((17, "payload-a".into()))
1494 );
1495 assert_eq!(cache.get_input_prediction("missing").unwrap(), None);
1496 cache
1497 .put_input_prediction("first", 18, None, "payload-c")
1498 .unwrap();
1499 assert_eq!(
1500 cache.get_input_prediction("first").unwrap(),
1501 Some((18, "payload-c".into()))
1502 );
1503 assert_eq!(
1504 cache.get_input_prediction("second").unwrap(),
1505 Some((42, "payload-b".into()))
1506 );
1507 }
1508
1509 fn prediction_last_used(cache: &FileHashCache<'_>, identity: &str) -> i64 {
1510 cache
1511 .db()
1512 .query_row(
1513 "SELECT last_used FROM input_predictions WHERE identity = ?1",
1514 params![identity],
1515 |row| row.get(0),
1516 )
1517 .unwrap()
1518 }
1519
1520 fn set_prediction_last_used(cache: &FileHashCache<'_>, identity: &str, last_used: i64) {
1521 cache
1522 .db()
1523 .execute(
1524 "UPDATE input_predictions SET last_used = ?2 WHERE identity = ?1",
1525 params![identity, last_used],
1526 )
1527 .unwrap();
1528 }
1529
1530 fn create_legacy_input_predictions(db: &Connection) {
1532 db.execute_batch(
1533 "CREATE TABLE input_predictions (
1534 identity TEXT PRIMARY KEY,
1535 schema INTEGER NOT NULL,
1536 crate_name TEXT,
1537 prediction_json TEXT NOT NULL,
1538 updated_at TEXT NOT NULL DEFAULT (datetime('now'))
1539 );
1540 INSERT INTO input_predictions (identity, schema, crate_name, prediction_json, updated_at)
1541 VALUES ('old-a', 3, 'alpha', 'payload-a', '2020-01-01 00:00:00'),
1542 ('old-b', 4, NULL, 'payload-b', '2020-01-02 00:00:00');",
1543 )
1544 .unwrap();
1545 }
1546
1547 #[test]
1548 fn input_prediction_windows_are_pinned() {
1549 assert_eq!(INPUT_PREDICTION_TOUCH_INTERVAL_SECS, 86_400);
1550 assert_eq!(INPUT_PREDICTION_RETENTION_SECS, 2_592_000);
1551 assert!(!prediction_touch_due(1_000_000, 1_000_000 + 86_399));
1552 assert!(prediction_touch_due(1_000_000, 1_000_000 + 86_400));
1553 assert!(!prediction_touch_due(2_000_000, 1_000_000));
1555 assert!(!prediction_touch_due(i64::MAX, i64::MIN));
1556 assert_eq!(prediction_prune_cutoff(3_000_000), 408_000);
1557 assert_eq!(prediction_prune_cutoff(i64::MIN), i64::MIN);
1558 }
1559
1560 #[test]
1561 fn input_prediction_hit_refreshes_last_used_at_most_once_per_interval() {
1562 let dir = tempfile::tempdir().unwrap();
1563 let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1564 cache
1565 .put_input_prediction("unit", 1, None, "payload")
1566 .unwrap();
1567 let now = 1_900_000_000;
1568 set_prediction_last_used(&cache, "unit", now - 86_399);
1569 assert_eq!(
1570 cache.get_input_prediction_at("unit", now).unwrap(),
1571 Some((1, "payload".into()))
1572 );
1573 assert_eq!(prediction_last_used(&cache, "unit"), now - 86_399);
1574
1575 set_prediction_last_used(&cache, "unit", now - 86_400);
1576 assert_eq!(
1577 cache.get_input_prediction_at("unit", now).unwrap(),
1578 Some((1, "payload".into()))
1579 );
1580 assert_eq!(prediction_last_used(&cache, "unit"), now);
1581
1582 cache.get_input_prediction_at("unit", now + 5).unwrap();
1584 assert_eq!(prediction_last_used(&cache, "unit"), now);
1585 cache
1587 .put_input_prediction("other", 1, None, "payload")
1588 .unwrap();
1589 set_prediction_last_used(&cache, "other", 7);
1590 assert_eq!(cache.get_input_prediction_at("absent", now).unwrap(), None);
1591 cache.get_input_prediction_at("unit", now + 86_400).unwrap();
1592 assert_eq!(prediction_last_used(&cache, "other"), 7);
1593 }
1594
1595 #[test]
1596 fn input_prediction_read_and_record_stamp_the_wall_clock() {
1597 let dir = tempfile::tempdir().unwrap();
1598 let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1599 let before = unix_now();
1600 assert!(before > 1_700_000_000);
1601 cache
1602 .put_input_prediction("unit", 1, None, "payload")
1603 .unwrap();
1604 assert!(prediction_last_used(&cache, "unit") >= before);
1605
1606 set_prediction_last_used(&cache, "unit", 1);
1607 cache.get_input_prediction("unit").unwrap();
1608 assert!(prediction_last_used(&cache, "unit") >= before);
1609
1610 set_prediction_last_used(&cache, "unit", 1);
1612 cache
1613 .put_input_prediction("unit", 2, None, "payload-2")
1614 .unwrap();
1615 assert!(prediction_last_used(&cache, "unit") >= before);
1616 }
1617
1618 #[test]
1619 fn prune_input_predictions_removes_only_rows_unused_past_the_window() {
1620 let dir = tempfile::tempdir().unwrap();
1621 let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1622 assert_eq!(cache.prune_input_predictions().unwrap(), 0);
1624
1625 let now = 1_900_000_000;
1626 for identity in ["expired", "boundary", "fresh"] {
1627 cache
1628 .put_input_prediction(identity, 1, None, "payload")
1629 .unwrap();
1630 }
1631 set_prediction_last_used(&cache, "expired", now - 2_592_001);
1632 set_prediction_last_used(&cache, "boundary", now - 2_592_000);
1633 set_prediction_last_used(&cache, "fresh", now);
1634 assert_eq!(cache.prune_input_predictions_at(now).unwrap(), 1);
1635 assert_eq!(cache.get_input_prediction_at("expired", now).unwrap(), None);
1636 assert!(
1637 cache
1638 .get_input_prediction_at("boundary", now - 86_400)
1639 .unwrap()
1640 .is_some()
1641 );
1642 assert!(
1643 cache
1644 .get_input_prediction_at("fresh", now)
1645 .unwrap()
1646 .is_some()
1647 );
1648 assert_eq!(cache.prune_input_predictions_at(now).unwrap(), 0);
1649
1650 let indexed: bool = cache
1651 .db()
1652 .query_row(
1653 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'index'
1654 AND name = 'input_predictions_last_used')",
1655 [],
1656 |row| row.get(0),
1657 )
1658 .unwrap();
1659 assert!(indexed);
1660 }
1661
1662 #[test]
1663 fn prune_input_predictions_uses_the_wall_clock() {
1664 let dir = tempfile::tempdir().unwrap();
1665 let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1666 cache
1667 .put_input_prediction("ancient", 1, None, "payload")
1668 .unwrap();
1669 cache
1670 .put_input_prediction("live", 1, None, "payload")
1671 .unwrap();
1672 set_prediction_last_used(&cache, "ancient", 1);
1673 assert_eq!(cache.prune_input_predictions().unwrap(), 1);
1674 assert!(cache.get_input_prediction("live").unwrap().is_some());
1675 assert_eq!(cache.get_input_prediction("ancient").unwrap(), None);
1676 }
1677
1678 #[test]
1679 fn schema_upgrade_adds_last_used_and_keeps_existing_predictions() {
1680 let dir = tempfile::tempdir().unwrap();
1681 let db_path = dir.path().join("index.db");
1682 let db = Connection::open(&db_path).unwrap();
1683 create_legacy_input_predictions(&db);
1684 assert!(!input_predictions_have_last_used(&db).unwrap());
1685
1686 ensure_input_predictions_last_used(&db, 1_700_000_000).unwrap();
1687 assert!(input_predictions_have_last_used(&db).unwrap());
1688 ensure_input_predictions_last_used(&db, 1_700_000_999).unwrap();
1690 ensure_file_hash_cache_schema(&db).unwrap();
1691 drop(db);
1692
1693 let cache = FileHashCache::open(&db_path).unwrap();
1694 assert_eq!(prediction_last_used(&cache, "old-a"), 1_700_000_000);
1696 assert_eq!(prediction_last_used(&cache, "old-b"), 1_700_000_000);
1697 assert_eq!(
1698 cache
1699 .get_input_prediction_at("old-a", 1_700_000_000)
1700 .unwrap(),
1701 Some((3, "payload-a".into()))
1702 );
1703 assert_eq!(
1704 cache
1705 .get_input_prediction_at("old-b", 1_700_000_000)
1706 .unwrap(),
1707 Some((4, "payload-b".into()))
1708 );
1709 assert_eq!(
1711 cache
1712 .prune_input_predictions_at(1_700_000_000 + 2_592_000)
1713 .unwrap(),
1714 0
1715 );
1716 assert_eq!(
1717 cache
1718 .prune_input_predictions_at(1_700_000_000 + 2_592_001)
1719 .unwrap(),
1720 2
1721 );
1722 cache
1724 .put_input_prediction("new", 5, None, "payload-n")
1725 .unwrap();
1726 assert!(prediction_last_used(&cache, "new") > 1_700_000_999);
1727 }
1728
1729 #[test]
1730 fn schema_upgrade_survives_many_connections_opening_at_once() {
1731 let dir = tempfile::tempdir().unwrap();
1732 let db_path = dir.path().join("index.db");
1733 {
1734 let db = Connection::open(&db_path).unwrap();
1735 db.pragma_update(None, "journal_mode", "WAL").unwrap();
1736 create_legacy_input_predictions(&db);
1737 }
1738 let barrier = std::sync::Arc::new(std::sync::Barrier::new(8));
1739 let workers: Vec<_> = (0..8)
1740 .map(|_| {
1741 let db_path = db_path.clone();
1742 let barrier = barrier.clone();
1743 std::thread::spawn(move || {
1744 let db = Connection::open(&db_path).unwrap();
1745 db.pragma_update(None, "busy_timeout", "5000").unwrap();
1746 barrier.wait();
1747 ensure_input_predictions_last_used(&db, 1_700_000_000)
1748 })
1749 })
1750 .collect();
1751 for worker in workers {
1752 worker.join().unwrap().unwrap();
1753 }
1754 let cache = FileHashCache::open(&db_path).unwrap();
1755 assert_eq!(prediction_last_used(&cache, "old-a"), 1_700_000_000);
1756 assert_eq!(prediction_last_used(&cache, "old-b"), 1_700_000_000);
1757 }
1758
1759 #[test]
1760 fn file_hash_memo_key_includes_inode() {
1761 let conn = rusqlite::Connection::open_in_memory().unwrap();
1765 ensure_file_hash_cache_schema(&conn).unwrap();
1766 let cache = FileHashCache::Borrowed(&conn);
1767
1768 let fp = |inode: i64| FileFingerprint {
1769 path: "/x/lib.rlib".to_string(),
1770 size: 100,
1771 mtime_ns: 1,
1772 ctime_ns: 2,
1773 inode,
1774 };
1775
1776 cache.put(&fp(10), "hash_for_inode_10").unwrap();
1777 assert_eq!(
1778 cache.get(&fp(10)).unwrap().as_deref(),
1779 Some("hash_for_inode_10")
1780 );
1781 assert_eq!(
1782 cache.get(&fp(20)).unwrap(),
1783 None,
1784 "a different inode (same path/size/mtime/ctime) must miss the memo"
1785 );
1786 }
1787
1788 #[test]
1789 fn test_hash_file() {
1790 let dir = tempfile::tempdir().unwrap();
1791 let file = dir.path().join("test.rs");
1792 std::fs::write(&file, b"fn main() {}").unwrap();
1793
1794 let hash = hash_file(&file).unwrap();
1795 assert_eq!(hash.len(), 64); let file2 = dir.path().join("test2.rs");
1799 std::fs::write(&file2, b"fn main() {}").unwrap();
1800 let hash2 = hash_file(&file2).unwrap();
1801 assert_eq!(hash, hash2);
1802
1803 let file3 = dir.path().join("test3.rs");
1805 std::fs::write(&file3, b"fn main() { println!(\"hello\"); }").unwrap();
1806 let hash3 = hash_file(&file3).unwrap();
1807 assert_ne!(hash, hash3);
1808
1809 let large = dir.path().join("large.rlib");
1812 let large_bytes: Vec<u8> = (0..256 * 1024 + 17)
1813 .map(|index| (index % 251) as u8)
1814 .collect();
1815 std::fs::write(&large, &large_bytes).unwrap();
1816 assert_eq!(
1817 hash_file(&large).unwrap(),
1818 blake3::hash(&large_bytes).to_hex().to_string()
1819 );
1820 }
1821
1822 #[test]
1823 fn lookup_cached_too_small_or_unreadable_is_uncacheable() {
1824 let dir = tempfile::tempdir().unwrap();
1828 let db_path = dir.path().join("index.db");
1829 let small = dir.path().join("small.rs");
1830 std::fs::write(&small, b"fn main() {}").unwrap();
1831
1832 let fh = FileHashCache::open(&db_path).unwrap();
1833 assert!(matches!(
1834 fh.lookup_cached(&small),
1835 FileHashLookup::Uncacheable
1836 ));
1837 assert!(matches!(
1839 fh.lookup_cached(&dir.path().join("nope.rlib")),
1840 FileHashLookup::Uncacheable
1841 ));
1842 }
1843
1844 #[test]
1845 fn lookup_cached_miss_then_record_then_hit_roundtrips() {
1846 let dir = tempfile::tempdir().unwrap();
1852 let db_path = dir.path().join("index.db");
1853 let file = dir.path().join("large.rlib");
1854 std::fs::write(&file, vec![7u8; 70 * 1024]).unwrap();
1855
1856 let fh = FileHashCache::open(&db_path).unwrap();
1857 let fp = match fh.lookup_cached(&file) {
1858 FileHashLookup::NeedsHash(fp) => fp,
1859 _ => panic!("expected NeedsHash on first lookup"),
1860 };
1861 fh.record_cached(&fp, "cafef00d");
1862
1863 match fh.lookup_cached(&file) {
1864 FileHashLookup::Hit(h) => assert_eq!(h, "cafef00d"),
1865 _ => panic!("expected Hit after record_cached"),
1866 }
1867 }
1868
1869 #[test]
1870 fn record_verified_honors_the_persistence_floor() {
1871 let dir = tempfile::tempdir().unwrap();
1872 let db_path = dir.path().join("index.db");
1873 let below = dir.path().join("below.rlib");
1874 let at_floor = dir.path().join("at-floor.rlib");
1875 std::fs::write(&below, vec![1u8; 65_535]).unwrap();
1876 std::fs::write(&at_floor, vec![2u8; 65_536]).unwrap();
1877
1878 let hasher = FileHashCache::open(&db_path).unwrap();
1879 hasher.record_verified(&FileFingerprint::from_path(&below).unwrap(), "below");
1880 hasher.record_verified(&FileFingerprint::from_path(&at_floor).unwrap(), "at-floor");
1881
1882 assert!(matches!(
1883 hasher.lookup_cached(&below),
1884 FileHashLookup::Uncacheable
1885 ));
1886 assert!(matches!(
1887 hasher.lookup_cached(&at_floor),
1888 FileHashLookup::Hit(hash) if hash == "at-floor"
1889 ));
1890
1891 let conn = rusqlite::Connection::open(&db_path).unwrap();
1892 let rows: i64 = conn
1893 .query_row("SELECT COUNT(*) FROM file_hashes", [], |row| row.get(0))
1894 .unwrap();
1895 assert_eq!(rows, 1, "sub-threshold fingerprints must not be stored");
1896 }
1897}