kache_store/
cc_memo.rs

1//! Shared C/C++ memo inputs. Artifact keys and input validation stay compiler-owned.
2
3use crate::file_hash::{FileFingerprint, FileHashCache, stamp_is_settled};
4use rusqlite::{Connection, Transaction, TransactionBehavior, params};
5
6#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
7pub struct CcPreprocessMemoInput {
8    pub name: String,
9    /// For a row read back whose stamp was never proven, `size` is -1,
10    /// which no file has.
11    #[serde(flatten)]
12    pub fingerprint: FileFingerprint,
13    pub content: String,
14    #[serde(default)]
15    pub mapped: String,
16    /// Wall clock read just before `fingerprint` was taken. 0 when unknown:
17    /// an input read back from the index, or one sent by an older wrapper.
18    #[serde(default)]
19    pub observed_ns: i64,
20}
21
22impl CcPreprocessMemoInput {
23    pub fn local_path(&self) -> &str {
24        &self.fingerprint.path
25    }
26
27    /// Whether `fingerprint` had settled when it was taken, so that any later
28    /// write moves it (see [`stamp_is_settled`]). Only such a stamp may stand
29    /// in for the content on a later lookup.
30    pub fn stamp_settled(&self) -> bool {
31        stamp_is_settled(&self.fingerprint, self.observed_ns)
32    }
33}
34
35/// The size an input row records in place of a stamp that had not settled
36/// when it was taken. No file has it, so no lookup matches the row by stamp
37/// and the content is compared instead. A row whose stamp has no matching
38/// proof reads back with it too.
39pub const UNPROVEN_SIZE: i64 = -1;
40
41/// The rule the C/C++ memo's rows are recorded under. Lookups serve a
42/// mapped hash or an assembler verdict only at this rule or a later one,
43/// and trust an input row's stamp only while the row's proof matches it.
44/// A release from before these columns writes neither, so nothing it
45/// records is served, before an upgrade or after it.
46///
47/// 1: a stamp is recorded only if it had settled when it was observed,
48///    and a mapped hash or a verdict only from the bytes behind its
49///    content hash.
50pub const CC_MEMO_RULE: i64 = 1;
51
52/// What an input row's `proof` holds for `stamp` under [`CC_MEMO_RULE`].
53///
54/// An older release records the same bytes under the same name by
55/// rewriting the shared row's stamp in place, and leaves `proof` alone. A
56/// stamp it wrote, possibly one taken moments after a write, then no longer
57/// matches the proof and proves nothing. One it rewrote unchanged still
58/// does: this release saw that stamp settle.
59fn stamp_proof(stamp: &FileFingerprint) -> i64 {
60    let mut hasher = blake3::Hasher::new();
61    hasher.update(&CC_MEMO_RULE.to_le_bytes());
62    hasher.update(stamp.path.as_bytes());
63    hasher.update(&[0]);
64    for value in [stamp.size, stamp.mtime_ns, stamp.ctime_ns, stamp.inode] {
65        hasher.update(&value.to_le_bytes());
66    }
67    let mut proof = [0; 8];
68    proof.copy_from_slice(&hasher.finalize().as_bytes()[..8]);
69    i64::from_le_bytes(proof)
70}
71
72/// The stamp an input row read back with no proof stands for: one that
73/// matches no file.
74fn unproven(path: String) -> FileFingerprint {
75    FileFingerprint {
76        path,
77        size: UNPROVEN_SIZE,
78        mtime_ns: 0,
79        ctime_ns: 0,
80        inode: 0,
81    }
82}
83
84#[derive(Debug, PartialEq, Eq)]
85pub struct CcPreprocessMemo {
86    pub preprocessed_hash: String,
87    pub inputs: Vec<CcPreprocessMemoInput>,
88    pub needs_touch: bool,
89}
90
91fn has_current_schema(db: &Connection) -> rusqlite::Result<bool> {
92    db.query_row(
93        "SELECT EXISTS(SELECT 1 FROM pragma_table_info('cc_preprocess_memos') WHERE name = 'input_count')",
94        [], |row| row.get(0),
95    )
96}
97
98pub(crate) fn ensure_schema(db: &Connection) -> rusqlite::Result<()> {
99    if has_current_schema(db)? {
100        return Ok(());
101    }
102    // Recheck after acquiring the writer lock: concurrent wrappers may have
103    // observed the legacy table before the first migration committed.
104    let tx = Transaction::new_unchecked(db, TransactionBehavior::Immediate)?;
105    if !has_current_schema(&tx)? {
106        tx.execute_batch(
107            "DROP TABLE IF EXISTS cc_preprocess_memos;
108             CREATE TABLE cc_preprocess_memos (
109                id INTEGER PRIMARY KEY,
110                memo_key TEXT NOT NULL UNIQUE,
111                preprocessed_hash TEXT NOT NULL,
112                input_count INTEGER NOT NULL,
113                last_used INTEGER NOT NULL DEFAULT (unixepoch())
114             );
115             CREATE INDEX cc_preprocess_memos_last_used ON cc_preprocess_memos(last_used);
116             CREATE TABLE cc_memo_inputs (
117                id INTEGER PRIMARY KEY,
118                name TEXT NOT NULL,
119                content TEXT NOT NULL,
120                mapped TEXT NOT NULL,
121                local_path TEXT NOT NULL,
122                size INTEGER NOT NULL,
123                mtime_ns INTEGER NOT NULL,
124                ctime_ns INTEGER NOT NULL,
125                inode INTEGER NOT NULL,
126                proof INTEGER,
127                UNIQUE(name, content, mapped)
128             );
129             CREATE TABLE cc_memo_input_refs (
130                memo_id INTEGER NOT NULL REFERENCES cc_preprocess_memos(id),
131                input_id INTEGER NOT NULL REFERENCES cc_memo_inputs(id),
132                PRIMARY KEY(memo_id, input_id)
133             ) WITHOUT ROWID;
134             CREATE INDEX cc_memo_input_refs_input ON cc_memo_input_refs(input_id);",
135        )?;
136    }
137    tx.commit()
138}
139
140/// The mapped-content memo: what a file's bytes hash to once the prefix maps
141/// of one invocation are applied. Keyed by the raw content hash and the map
142/// set, never by path, so a header shared by two hundred translation units
143/// is read and rewritten once per map set instead of once per unit.
144pub(crate) fn ensure_mapped_hash_schema(db: &Connection) -> rusqlite::Result<()> {
145    db.execute_batch(
146        "CREATE TABLE IF NOT EXISTS cc_mapped_hashes (
147            content TEXT NOT NULL,
148            maps TEXT NOT NULL,
149            mapped TEXT NOT NULL,
150            rule INTEGER NOT NULL DEFAULT 0,
151            PRIMARY KEY(content, maps)
152         ) WITHOUT ROWID;
153         CREATE TABLE IF NOT EXISTS cc_asm_scans (
154            content TEXT PRIMARY KEY,
155            construct TEXT NOT NULL,
156            rule INTEGER NOT NULL DEFAULT 0
157         ) WITHOUT ROWID;",
158    )
159}
160
161/// The columns [`CC_MEMO_RULE`] reads, as `ALTER TABLE` adds them to a
162/// table from before them.
163const RULE_COLUMNS: [(&str, &str, &str); 3] = [
164    ("cc_memo_inputs", "proof", "proof INTEGER"),
165    (
166        "cc_mapped_hashes",
167        "rule",
168        "rule INTEGER NOT NULL DEFAULT 0",
169    ),
170    ("cc_asm_scans", "rule", "rule INTEGER NOT NULL DEFAULT 0"),
171];
172
173fn has_column(db: &Connection, table: &str, column: &str) -> rusqlite::Result<bool> {
174    db.query_row(
175        "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
176        params![table, column],
177        |row| row.get(0),
178    )
179}
180
181/// Whether every column [`CC_MEMO_RULE`] reads is present.
182pub(crate) fn has_rule_columns(db: &Connection) -> rusqlite::Result<bool> {
183    for (table, column, _) in RULE_COLUMNS {
184        if !has_column(db, table, column)? {
185            return Ok(false);
186        }
187    }
188    Ok(true)
189}
190
191/// Add the columns [`CC_MEMO_RULE`] reads to tables from before them, and
192/// [`purge`] the memo once as they arrive. Checked again under the write
193/// lock, so the purge runs once however many processes open the index
194/// together. Gated on the columns rather than the index generation, which
195/// an older release stamps back over a newer one each time it opens the
196/// index; no older release drops these columns.
197pub(crate) fn ensure_rule(db: &Connection) -> rusqlite::Result<()> {
198    if has_rule_columns(db)? {
199        return Ok(());
200    }
201    add_rule_columns(db)
202}
203
204/// The part of [`ensure_rule`] that runs under the write lock, once a look
205/// without it found a column missing. Another process may have added the
206/// columns since and recorded rows under them, so it looks again before it
207/// purges.
208fn add_rule_columns(db: &Connection) -> rusqlite::Result<()> {
209    let tx = Transaction::new_unchecked(db, TransactionBehavior::Immediate)?;
210    if !has_rule_columns(&tx)? {
211        for (table, column, definition) in RULE_COLUMNS {
212            if !has_column(&tx, table, column)? {
213                tx.execute_batch(&format!("ALTER TABLE {table} ADD COLUMN {definition}"))?;
214            }
215        }
216        purge(&tx)?;
217    }
218    tx.commit()
219}
220
221/// Forget every C/C++ memo, with the mapped hashes and assembler verdicts
222/// beside them. [`ensure_rule`] runs this once for rows an older kache
223/// recorded: it trusted a stamp however soon after a write it was taken,
224/// and it learned a mapped hash or a verdict from a second read without
225/// checking that read saw the bytes behind the content hash.
226fn purge(db: &Connection) -> rusqlite::Result<()> {
227    db.execute_batch(
228        "DELETE FROM cc_memo_input_refs;
229         DELETE FROM cc_preprocess_memos;
230         DELETE FROM cc_memo_inputs;
231         DELETE FROM cc_mapped_hashes;
232         DELETE FROM cc_asm_scans;",
233    )
234}
235
236/// Headers looked up per statement. Far below SQLite's bound on bound
237/// parameters, and a round number of prepared-statement shapes to cache.
238const MEMO_LOOKUP_CHUNK: usize = 256;
239
240impl FileHashCache<'_> {
241    /// Memoised mapped hashes for `contents` under the map set `maps`,
242    /// recorded under `CC_MEMO_RULE` or a later rule.
243    pub fn get_cc_mapped_hashes(
244        &self,
245        maps: &str,
246        contents: &[&str],
247    ) -> rusqlite::Result<std::collections::HashMap<String, String>> {
248        let mut found = std::collections::HashMap::new();
249        // One statement per chunk instead of one per header: a translation
250        // unit reads a hundred or more, and the per-statement cost was most
251        // of this lookup.
252        for chunk in contents.chunks(MEMO_LOOKUP_CHUNK) {
253            let placeholders = vec!["?"; chunk.len()].join(",");
254            let mut stmt = self.db().prepare_cached(&format!(
255                "SELECT content, mapped FROM cc_mapped_hashes
256                 WHERE maps = ?1 AND content IN ({placeholders}) AND rule >= {CC_MEMO_RULE}"
257            ))?;
258            let args = std::iter::once(maps).chain(chunk.iter().copied());
259            let rows = stmt.query_map(rusqlite::params_from_iter(args), |row| {
260                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
261            })?;
262            for row in rows {
263                let (content, mapped) = row?;
264                found.insert(content, mapped);
265            }
266        }
267        Ok(found)
268    }
269
270    /// Memoised assembler scans for `contents`: the construct found, or an
271    /// empty string for a clean file. Only verdicts recorded under
272    /// `CC_MEMO_RULE` or a later rule count.
273    pub fn get_cc_asm_scans(
274        &self,
275        contents: &[&str],
276    ) -> rusqlite::Result<std::collections::HashMap<String, String>> {
277        let mut found = std::collections::HashMap::new();
278        for chunk in contents.chunks(MEMO_LOOKUP_CHUNK) {
279            let placeholders = vec!["?"; chunk.len()].join(",");
280            let mut stmt = self.db().prepare_cached(&format!(
281                "SELECT content, construct FROM cc_asm_scans
282                 WHERE content IN ({placeholders}) AND rule >= {CC_MEMO_RULE}"
283            ))?;
284            let rows = stmt
285                .query_map(rusqlite::params_from_iter(chunk.iter().copied()), |row| {
286                    Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
287                })?;
288            for row in rows {
289                let (content, construct) = row?;
290                found.insert(content, construct);
291            }
292        }
293        Ok(found)
294    }
295
296    /// Record assembler scans from this invocation, in one transaction. A
297    /// verdict an older rule recorded is replaced.
298    pub fn put_cc_asm_scans(&self, pairs: &[(String, String)]) -> rusqlite::Result<()> {
299        if pairs.is_empty() {
300            return Ok(());
301        }
302        let tx = Transaction::new_unchecked(self.db(), TransactionBehavior::Immediate)?;
303        {
304            let mut put = tx.prepare_cached(&format!(
305                "INSERT INTO cc_asm_scans(content, construct, rule) VALUES (?1, ?2, {CC_MEMO_RULE})
306                 ON CONFLICT(content) DO UPDATE SET
307                 construct = excluded.construct, rule = excluded.rule
308                 WHERE cc_asm_scans.rule < excluded.rule"
309            ))?;
310            for (content, construct) in pairs {
311                put.execute(params![content, construct])?;
312            }
313        }
314        tx.commit()
315    }
316
317    /// Record mapped hashes computed this invocation, in one transaction. A
318    /// mapped hash an older rule recorded is replaced.
319    pub fn put_cc_mapped_hashes(
320        &self,
321        maps: &str,
322        pairs: &[(String, String)],
323    ) -> rusqlite::Result<()> {
324        if pairs.is_empty() {
325            return Ok(());
326        }
327        let tx = Transaction::new_unchecked(self.db(), TransactionBehavior::Immediate)?;
328        {
329            let mut put = tx.prepare_cached(&format!(
330                "INSERT INTO cc_mapped_hashes(content, maps, mapped, rule)
331                 VALUES (?1, ?2, ?3, {CC_MEMO_RULE})
332                 ON CONFLICT(content, maps) DO UPDATE SET
333                 mapped = excluded.mapped, rule = excluded.rule
334                 WHERE cc_mapped_hashes.rule < excluded.rule"
335            ))?;
336            for (content, mapped) in pairs {
337                put.execute(params![content, maps, mapped])?;
338            }
339        }
340        tx.commit()
341    }
342
343    pub fn get_cc_preprocess_memo(
344        &self,
345        memo_key: &str,
346    ) -> rusqlite::Result<Option<CcPreprocessMemo>> {
347        // One statement provides a single SQLite read snapshot. Reading the
348        // digest separately could pair an old digest with a replacement's inputs.
349        let mut stmt = self.db().prepare_cached(
350            "SELECT m.preprocessed_hash, m.input_count, m.last_used <= unixepoch() - 86400,
351                    i.id, i.name, i.content, i.mapped, i.local_path,
352                    i.size, i.mtime_ns, i.ctime_ns, i.inode, i.proof
353             FROM cc_preprocess_memos m
354             LEFT JOIN cc_memo_input_refs r ON r.memo_id = m.id
355             LEFT JOIN cc_memo_inputs i ON i.id = r.input_id
356             WHERE m.memo_key = ?1 ORDER BY r.input_id",
357        )?;
358        let mut rows = stmt.query([memo_key])?;
359        let mut result = None;
360        let mut expected_count = 0;
361        while let Some(row) = rows.next()? {
362            if row.get::<_, Option<i64>>(3)?.is_none() {
363                return Ok(None);
364            }
365            if result.is_none() {
366                expected_count = row.get::<_, i64>(1)?;
367                result = Some(CcPreprocessMemo {
368                    preprocessed_hash: row.get(0)?,
369                    inputs: Vec::new(),
370                    needs_touch: row.get(2)?,
371                });
372            }
373            let stamp = FileFingerprint {
374                path: row.get(7)?,
375                size: row.get(8)?,
376                mtime_ns: row.get(9)?,
377                ctime_ns: row.get(10)?,
378                inode: row.get(11)?,
379            };
380            let proven = row.get::<_, Option<i64>>(12)? == Some(stamp_proof(&stamp));
381            result.as_mut().unwrap().inputs.push(CcPreprocessMemoInput {
382                name: row.get(4)?,
383                content: row.get(5)?,
384                mapped: row.get(6)?,
385                fingerprint: if proven { stamp } else { unproven(stamp.path) },
386                observed_ns: 0,
387            });
388        }
389        // A damaged/incomplete reference set cannot stand in for the full closure.
390        Ok(result.filter(|memo| expected_count > 0 && memo.inputs.len() as i64 == expected_count))
391    }
392
393    /// Record `inputs` as the read set behind `preprocessed_hash`.
394    ///
395    /// Input rows are shared by every memo that read the same bytes under
396    /// the same name. An input whose stamp had settled when it was observed
397    /// gives the row its stamp, and the proof that lets a lookup trust it.
398    /// One observed sooner never does: a second write in the same timestamp
399    /// tick keeps the stamp and changes the bytes, so the row keeps the
400    /// stamp it has, or starts with none.
401    pub fn put_cc_preprocess_memo_inputs(
402        &self,
403        memo_key: &str,
404        preprocessed_hash: &str,
405        inputs: &[CcPreprocessMemoInput],
406    ) -> rusqlite::Result<()> {
407        let tx = Transaction::new_unchecked(self.db(), TransactionBehavior::Immediate)?;
408        let id: i64 = tx.query_row(
409            "INSERT INTO cc_preprocess_memos(memo_key, preprocessed_hash, input_count)
410             VALUES (?1, ?2, 0) ON CONFLICT(memo_key) DO UPDATE SET
411             preprocessed_hash = excluded.preprocessed_hash, last_used = unixepoch()
412             RETURNING id",
413            params![memo_key, preprocessed_hash],
414            |row| row.get(0),
415        )?;
416        tx.execute("DELETE FROM cc_memo_input_refs WHERE memo_id = ?1", [id])?;
417        {
418            let mut put = tx.prepare_cached(
419                "INSERT INTO cc_memo_inputs(name, content, mapped, local_path, size, mtime_ns, ctime_ns, inode, proof)
420                 VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
421                 ON CONFLICT(name, content, mapped) DO UPDATE SET
422                 local_path = excluded.local_path, size = excluded.size,
423                 mtime_ns = excluded.mtime_ns, ctime_ns = excluded.ctime_ns, inode = excluded.inode,
424                 proof = excluded.proof
425                 RETURNING id",
426            )?;
427            let mut put_unproven = tx.prepare_cached(
428                "INSERT INTO cc_memo_inputs(name, content, mapped, local_path, size, mtime_ns, ctime_ns, inode)
429                 VALUES (?1, ?2, ?3, ?4, ?5, 0, 0, 0)
430                 ON CONFLICT(name, content, mapped) DO UPDATE SET name = excluded.name
431                 RETURNING id",
432            )?;
433            let mut reference = tx.prepare_cached(
434                "INSERT OR IGNORE INTO cc_memo_input_refs(memo_id, input_id) VALUES (?1, ?2)",
435            )?;
436            for input in inputs {
437                let stamp = &input.fingerprint;
438                let input_id: i64 = if input.stamp_settled() {
439                    put.query_row(
440                        params![
441                            input.name,
442                            input.content,
443                            input.mapped,
444                            stamp.path,
445                            stamp.size,
446                            stamp.mtime_ns,
447                            stamp.ctime_ns,
448                            stamp.inode,
449                            stamp_proof(stamp)
450                        ],
451                        |row| row.get(0),
452                    )?
453                } else {
454                    put_unproven.query_row(
455                        params![
456                            input.name,
457                            input.content,
458                            input.mapped,
459                            stamp.path,
460                            UNPROVEN_SIZE
461                        ],
462                        |row| row.get(0),
463                    )?
464                };
465                reference.execute(params![id, input_id])?;
466            }
467        }
468        tx.execute(
469            "UPDATE cc_preprocess_memos SET input_count =
470             (SELECT count(*) FROM cc_memo_input_refs WHERE memo_id = ?1) WHERE id = ?1",
471            [id],
472        )?;
473        tx.commit()
474    }
475
476    /// Compatibility for existing compiler validation fixtures; runtime writes
477    /// and reads use typed inputs and never persist a JSON payload.
478    #[cfg(any(test, feature = "test-support"))]
479    pub fn put_cc_preprocess_memo(
480        &self,
481        memo_key: &str,
482        hash: &str,
483        json: &str,
484    ) -> rusqlite::Result<()> {
485        let inputs: Vec<CcPreprocessMemoInput> = serde_json::from_str(json)
486            .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?;
487        self.put_cc_preprocess_memo_inputs(memo_key, hash, &inputs)
488    }
489
490    /// Called after a successful validation and only when the read marked it
491    /// due. The SQL guard also bounds updates by competing processes.
492    pub fn touch_cc_preprocess_memo(&self, memo_key: &str) -> rusqlite::Result<usize> {
493        self.db().execute(
494            "UPDATE cc_preprocess_memos SET last_used = unixepoch()
495             WHERE memo_key = ?1 AND last_used <= unixepoch() - 86400",
496            [memo_key],
497        )
498    }
499
500    pub fn prune_cc_preprocess_memos(&self) -> rusqlite::Result<(usize, usize)> {
501        let tx = Transaction::new_unchecked(self.db(), TransactionBehavior::Immediate)?;
502        tx.execute(
503            "DELETE FROM cc_memo_input_refs WHERE memo_id IN
504             (SELECT id FROM cc_preprocess_memos WHERE last_used < unixepoch() - 2592000)",
505            [],
506        )?;
507        let memos = tx.execute(
508            "DELETE FROM cc_preprocess_memos WHERE last_used < unixepoch() - 2592000",
509            [],
510        )?;
511        let inputs = tx.execute(
512            "DELETE FROM cc_memo_inputs WHERE NOT EXISTS
513             (SELECT 1 FROM cc_memo_input_refs WHERE input_id = cc_memo_inputs.id)",
514            [],
515        )?;
516        tx.commit()?;
517        Ok((memos, inputs))
518    }
519}
520
521#[cfg(test)]
522mod tests {
523    use super::*;
524    use crate::file_hash::HASH_SETTLE_NS;
525
526    /// Observed long after any stamp these tests write, so they count as
527    /// settled.
528    const SETTLED: i64 = 10 * HASH_SETTLE_NS;
529
530    fn input(name: &str, content: &str) -> CcPreprocessMemoInput {
531        CcPreprocessMemoInput {
532            name: name.into(),
533            content: content.into(),
534            mapped: format!("mapped-{content}"),
535            fingerprint: FileFingerprint {
536                path: format!("/checkout/{name}"),
537                size: 10,
538                mtime_ns: 20,
539                ctime_ns: 30,
540                inode: 40,
541            },
542            observed_ns: SETTLED,
543        }
544    }
545
546    /// `input` as a lookup reads it back: the index keeps no observation
547    /// time.
548    fn read_back(mut input: CcPreprocessMemoInput) -> CcPreprocessMemoInput {
549        input.observed_ns = 0;
550        input
551    }
552
553    fn count(cache: &FileHashCache<'_>, table: &str) -> i64 {
554        cache
555            .db()
556            .query_row(&format!("SELECT count(*) FROM {table}"), [], |row| {
557                row.get(0)
558            })
559            .unwrap()
560    }
561
562    /// Every process that opens an older index looks for the rule columns
563    /// before it takes the write lock, and each that finds one missing goes
564    /// on to take it. Only the first may purge: by the time a later one holds
565    /// the lock, the columns are there and the rows under them are current.
566    #[test]
567    fn the_purge_runs_once_however_many_openers_found_a_column_missing() {
568        const MEMO_TABLES: [&str; 5] = [
569            "cc_preprocess_memos",
570            "cc_memo_inputs",
571            "cc_memo_input_refs",
572            "cc_mapped_hashes",
573            "cc_asm_scans",
574        ];
575        let dir = tempfile::tempdir().unwrap();
576        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
577        let record = |cache: &FileHashCache<'_>| {
578            cache
579                .put_cc_preprocess_memo_inputs("memo", "hash", &[input("a.h", "one")])
580                .unwrap();
581            cache
582                .put_cc_mapped_hashes("maps", &[("one".into(), "mapped".into())])
583                .unwrap();
584            cache
585                .put_cc_asm_scans(&[("one".into(), String::new())])
586                .unwrap();
587        };
588        let rows = |cache: &FileHashCache<'_>| MEMO_TABLES.map(|table| count(cache, table));
589        record(&cache);
590        cache
591            .db()
592            .execute_batch(
593                "ALTER TABLE cc_memo_inputs DROP COLUMN proof;
594                 ALTER TABLE cc_mapped_hashes DROP COLUMN rule;
595                 ALTER TABLE cc_asm_scans DROP COLUMN rule;",
596            )
597            .unwrap();
598        assert!(!has_rule_columns(cache.db()).unwrap());
599
600        // The first opener to take the lock adds the columns and purges.
601        ensure_rule(cache.db()).unwrap();
602        assert!(has_rule_columns(cache.db()).unwrap());
603        assert_eq!(rows(&cache), [0; 5]);
604
605        // A second opener found a column missing before that, and takes the
606        // lock only now, after rows were recorded under the columns.
607        record(&cache);
608        assert_eq!(rows(&cache), [1; 5]);
609        add_rule_columns(cache.db()).unwrap();
610        assert_eq!(rows(&cache), [1; 5], "the purge runs once");
611    }
612
613    #[test]
614    fn cc_memos_share_inputs_refresh_stamps_and_replace_one_key() {
615        let dir = tempfile::tempdir().unwrap();
616        let path = dir.path().join("index.db");
617        let cache = FileHashCache::open(&path).unwrap();
618        let first = input("shared.h", "one");
619        cache
620            .put_cc_preprocess_memo_inputs("a", "hash-a", std::slice::from_ref(&first))
621            .unwrap();
622        let mut moved = first.clone();
623        moved.fingerprint = FileFingerprint {
624            path: "/other/shared.h".into(),
625            size: 11,
626            mtime_ns: 21,
627            ctime_ns: 31,
628            inode: 41,
629        };
630        cache
631            .put_cc_preprocess_memo_inputs("b", "hash-b", std::slice::from_ref(&moved))
632            .unwrap();
633        assert_eq!(count(&cache, "cc_memo_inputs"), 1);
634        assert_eq!(count(&cache, "cc_memo_input_refs"), 2);
635        let memo = cache.get_cc_preprocess_memo("a").unwrap().unwrap();
636        assert_eq!(memo.preprocessed_hash, "hash-a");
637        assert_eq!(memo.inputs, vec![read_back(moved.clone())]);
638        assert!(!memo.needs_touch);
639        let second = input("shared.h", "two");
640        cache
641            .put_cc_preprocess_memo_inputs("a", "hash-c", std::slice::from_ref(&second))
642            .unwrap();
643        drop(cache);
644        let cache = FileHashCache::open(&path).unwrap();
645        assert_eq!(
646            cache.get_cc_preprocess_memo("a").unwrap().unwrap().inputs,
647            vec![read_back(second)]
648        );
649        let b = cache.get_cc_preprocess_memo("b").unwrap().unwrap();
650        assert_eq!(b.preprocessed_hash, "hash-b");
651        assert_eq!(b.inputs, vec![read_back(moved)]);
652        assert_eq!(count(&cache, "cc_memo_inputs"), 2);
653        assert!(cache.get_cc_preprocess_memo("absent").unwrap().is_none());
654    }
655
656    /// A stamp taken less than the settle window after its file changed is
657    /// never recorded: the memo keeps the input's bytes and no stamp, and a
658    /// row another memo proved keeps its own stamp.
659    #[test]
660    fn an_unsettled_stamp_is_never_recorded_nor_shared() {
661        let dir = tempfile::tempdir().unwrap();
662        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
663        let just_before_settling = |input: &CcPreprocessMemoInput| {
664            let changed = input.fingerprint.mtime_ns.max(input.fingerprint.ctime_ns);
665            changed + HASH_SETTLE_NS - 1
666        };
667
668        let mut fresh = input("fresh.h", "f");
669        fresh.observed_ns = just_before_settling(&fresh);
670        cache
671            .put_cc_preprocess_memo_inputs("fresh", "h", std::slice::from_ref(&fresh))
672            .unwrap();
673        let recorded = cache.get_cc_preprocess_memo("fresh").unwrap().unwrap();
674        let recorded = &recorded.inputs[0];
675        assert_eq!(recorded.fingerprint.size, UNPROVEN_SIZE);
676        assert_eq!(recorded.fingerprint.path, fresh.fingerprint.path);
677        assert_eq!(
678            (&recorded.content, &recorded.mapped),
679            (&fresh.content, &fresh.mapped)
680        );
681
682        let proven = input("shared.h", "s");
683        cache
684            .put_cc_preprocess_memo_inputs("a", "h", std::slice::from_ref(&proven))
685            .unwrap();
686        let mut elsewhere = proven.clone();
687        elsewhere.fingerprint = FileFingerprint {
688            path: "/other/shared.h".into(),
689            size: 10,
690            mtime_ns: 50,
691            ctime_ns: 60,
692            inode: 41,
693        };
694        elsewhere.observed_ns = just_before_settling(&elsewhere);
695        cache
696            .put_cc_preprocess_memo_inputs("b", "h", std::slice::from_ref(&elsewhere))
697            .unwrap();
698        assert_eq!(count(&cache, "cc_memo_inputs"), 2, "one row per bytes");
699        for key in ["a", "b"] {
700            assert_eq!(
701                cache.get_cc_preprocess_memo(key).unwrap().unwrap().inputs,
702                vec![read_back(proven.clone())],
703                "{key}: the unsettled stamp replaced the proven one"
704            );
705        }
706
707        // One tick later the same observation is proof and takes the row.
708        elsewhere.observed_ns += 1;
709        cache
710            .put_cc_preprocess_memo_inputs("b", "h", std::slice::from_ref(&elsewhere))
711            .unwrap();
712        assert_eq!(
713            cache.get_cc_preprocess_memo("a").unwrap().unwrap().inputs,
714            vec![read_back(elsewhere)]
715        );
716    }
717
718    /// No file can match a row recorded without a stamp: not by the stamp a
719    /// lookup reads back, nor by the columns an older release reads without
720    /// the proof. Windows reports inode 0, and 0 for times before 1970, so
721    /// only the size rules such a file out, and no file's size is negative.
722    #[test]
723    fn a_row_recorded_without_a_stamp_matches_no_file() {
724        let dir = tempfile::tempdir().unwrap();
725        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
726        let mut unsettled = input("fresh.h", "f");
727        // Observed the moment it last changed.
728        unsettled.observed_ns = unsettled.fingerprint.ctime_ns;
729        cache
730            .put_cc_preprocess_memo_inputs("fresh", "h", std::slice::from_ref(&unsettled))
731            .unwrap();
732        let memo = cache.get_cc_preprocess_memo("fresh").unwrap().unwrap();
733        let columns = cache
734            .db()
735            .query_row(
736                "SELECT local_path, size, mtime_ns, ctime_ns, inode FROM cc_memo_inputs",
737                [],
738                |row| {
739                    Ok(FileFingerprint {
740                        path: row.get(0)?,
741                        size: row.get(1)?,
742                        mtime_ns: row.get(2)?,
743                        ctime_ns: row.get(3)?,
744                        inode: row.get(4)?,
745                    })
746                },
747            )
748            .unwrap();
749        for stamp in [&memo.inputs[0].fingerprint, &columns] {
750            assert!(stamp.size < 0, "{stamp:?}");
751        }
752    }
753
754    /// A hand-off from a wrapper that predates observation times carries
755    /// none, and its stamps count as unsettled.
756    #[test]
757    fn an_input_without_an_observation_time_is_unsettled() {
758        let mut json = serde_json::to_value(input("a.h", "a")).unwrap();
759        json.as_object_mut().unwrap().remove("observed_ns");
760        let old: CcPreprocessMemoInput = serde_json::from_value(json).unwrap();
761        assert_eq!(old.observed_ns, 0);
762        assert!(!old.stamp_settled());
763        assert!(input("a.h", "a").stamp_settled());
764    }
765
766    /// How 1.0.0 records a memo: it rewrites the stamp of each input row
767    /// its bytes share and names no proof.
768    fn record_as_older_release(
769        cache: &FileHashCache<'_>,
770        memo_key: &str,
771        inputs: &[CcPreprocessMemoInput],
772    ) {
773        let db = cache.db();
774        let memo: i64 = db
775            .query_row(
776                "INSERT INTO cc_preprocess_memos(memo_key, preprocessed_hash, input_count)
777                 VALUES (?1, 'older', 0) ON CONFLICT(memo_key) DO UPDATE SET
778                 preprocessed_hash = excluded.preprocessed_hash, last_used = unixepoch()
779                 RETURNING id",
780                [memo_key],
781                |row| row.get(0),
782            )
783            .unwrap();
784        db.execute("DELETE FROM cc_memo_input_refs WHERE memo_id = ?1", [memo])
785            .unwrap();
786        for input in inputs {
787            let stamp = &input.fingerprint;
788            let id: i64 = db
789                .query_row(
790                    "INSERT INTO cc_memo_inputs(name, content, mapped, local_path, size, mtime_ns, ctime_ns, inode)
791                     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
792                     ON CONFLICT(name, content, mapped) DO UPDATE SET
793                     local_path = excluded.local_path, size = excluded.size,
794                     mtime_ns = excluded.mtime_ns, ctime_ns = excluded.ctime_ns, inode = excluded.inode
795                     RETURNING id",
796                    params![
797                        input.name,
798                        input.content,
799                        input.mapped,
800                        stamp.path,
801                        stamp.size,
802                        stamp.mtime_ns,
803                        stamp.ctime_ns,
804                        stamp.inode
805                    ],
806                    |row| row.get(0),
807                )
808                .unwrap();
809            db.execute(
810                "INSERT OR IGNORE INTO cc_memo_input_refs(memo_id, input_id) VALUES (?1, ?2)",
811                params![memo, id],
812            )
813            .unwrap();
814        }
815        db.execute(
816            "UPDATE cc_preprocess_memos SET input_count =
817             (SELECT count(*) FROM cc_memo_input_refs WHERE memo_id = ?1) WHERE id = ?1",
818            [memo],
819        )
820        .unwrap();
821    }
822
823    /// An older release that records the same bytes rewrites the shared
824    /// row's stamp in place. A stamp it changed proves nothing until this
825    /// release records one again; one it rewrote unchanged still does. A row
826    /// only the older release wrote never proves its input.
827    #[test]
828    fn a_stamp_an_older_release_rewrote_proves_nothing() {
829        let dir = tempfile::tempdir().unwrap();
830        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
831        let stamp_of = |memo_key: &str| {
832            cache
833                .get_cc_preprocess_memo(memo_key)
834                .unwrap()
835                .unwrap()
836                .inputs[0]
837                .fingerprint
838                .clone()
839        };
840        let proven = input("shared.h", "s");
841        cache
842            .put_cc_preprocess_memo_inputs("a", "h", std::slice::from_ref(&proven))
843            .unwrap();
844        assert_eq!(stamp_of("a"), proven.fingerprint);
845
846        record_as_older_release(&cache, "older", std::slice::from_ref(&proven));
847        assert_eq!(stamp_of("a"), proven.fingerprint, "rewritten unchanged");
848
849        for field in 0..5 {
850            cache
851                .put_cc_preprocess_memo_inputs("a", "h", std::slice::from_ref(&proven))
852                .unwrap();
853            assert_eq!(stamp_of("a"), proven.fingerprint, "field {field}");
854            let mut rewritten = proven.clone();
855            let stamp = &mut rewritten.fingerprint;
856            match field {
857                0 => stamp.path = "/other/shared.h".into(),
858                1 => stamp.size += 1,
859                2 => stamp.mtime_ns += 1,
860                3 => stamp.ctime_ns += 1,
861                _ => stamp.inode += 1,
862            }
863            record_as_older_release(&cache, "older", std::slice::from_ref(&rewritten));
864            assert_eq!(
865                stamp_of("a"),
866                unproven(rewritten.fingerprint.path.clone()),
867                "field {field}"
868            );
869            cache
870                .put_cc_preprocess_memo_inputs("a", "h", std::slice::from_ref(&rewritten))
871                .unwrap();
872            assert_eq!(stamp_of("a"), rewritten.fingerprint, "field {field}");
873        }
874
875        let other = input("other.h", "o");
876        record_as_older_release(&cache, "older", std::slice::from_ref(&other));
877        assert_eq!(stamp_of("older"), unproven(other.fingerprint.path));
878    }
879
880    /// A mapped hash or verdict an older release recorded is never served,
881    /// and this release's put replaces it. The older release only ever
882    /// inserts, so it leaves what this release recorded alone.
883    #[test]
884    fn a_mapped_hash_or_verdict_an_older_release_recorded_is_never_served() {
885        let dir = tempfile::tempdir().unwrap();
886        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
887        // The statements 1.0.0 records them with.
888        let record_as_older_release = |value: &str| {
889            cache
890                .db()
891                .execute(
892                    "INSERT OR IGNORE INTO cc_mapped_hashes(content, maps, mapped)
893                     VALUES ('c', 'maps', ?1)",
894                    [value],
895                )
896                .unwrap();
897            cache
898                .db()
899                .execute(
900                    "INSERT OR IGNORE INTO cc_asm_scans(content, construct) VALUES ('c', ?1)",
901                    [value],
902                )
903                .unwrap();
904        };
905        record_as_older_release("older");
906        assert!(
907            cache
908                .get_cc_mapped_hashes("maps", &["c"])
909                .unwrap()
910                .is_empty()
911        );
912        assert!(cache.get_cc_asm_scans(&["c"]).unwrap().is_empty());
913
914        cache
915            .put_cc_mapped_hashes("maps", &[("c".into(), "current".into())])
916            .unwrap();
917        cache
918            .put_cc_asm_scans(&[("c".into(), String::new())])
919            .unwrap();
920        record_as_older_release("older again");
921        let mapped = cache.get_cc_mapped_hashes("maps", &["c"]).unwrap();
922        assert_eq!(mapped.get("c").map(String::as_str), Some("current"));
923        let verdicts = cache.get_cc_asm_scans(&["c"]).unwrap();
924        assert_eq!(verdicts.get("c").map(String::as_str), Some(""));
925    }
926
927    #[test]
928    fn cc_memo_uniqueness_includes_name_and_both_hashes() {
929        let dir = tempfile::tempdir().unwrap();
930        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
931        let base = input("a.h", "raw");
932        let mut different_map = base.clone();
933        different_map.mapped = "other".into();
934        let inputs = vec![
935            base.clone(),
936            input("b.h", "raw"),
937            input("a.h", "changed"),
938            different_map,
939            base,
940        ];
941        cache
942            .put_cc_preprocess_memo_inputs("unit", "hash", &inputs)
943            .unwrap();
944        assert_eq!(count(&cache, "cc_memo_inputs"), 4);
945        assert_eq!(count(&cache, "cc_memo_input_refs"), 4);
946        assert_eq!(
947            cache
948                .get_cc_preprocess_memo("unit")
949                .unwrap()
950                .unwrap()
951                .inputs
952                .len(),
953            4
954        );
955    }
956
957    /// The mapped-hash memo answers by content and map set, the assembler
958    /// scan memo by content; both keep what was put across a reopen, and a
959    /// duplicate put leaves the first value in place.
960    #[test]
961    fn mapped_hash_and_asm_scan_memos_round_trip() {
962        let dir = tempfile::tempdir().unwrap();
963        let path = dir.path().join("index.db");
964        let cache = FileHashCache::open(&path).unwrap();
965        cache
966            .put_cc_mapped_hashes(
967                "maps-a",
968                &[
969                    ("c1".to_string(), "m1".to_string()),
970                    ("c2".to_string(), "m2".to_string()),
971                ],
972            )
973            .unwrap();
974        cache.put_cc_mapped_hashes("maps-a", &[]).unwrap();
975        cache
976            .put_cc_asm_scans(&[
977                ("c1".to_string(), String::new()),
978                ("c3".to_string(), ".incbin".to_string()),
979            ])
980            .unwrap();
981        cache.put_cc_asm_scans(&[]).unwrap();
982        drop(cache);
983        let cache = FileHashCache::open(&path).unwrap();
984        let mapped = cache
985            .get_cc_mapped_hashes("maps-a", &["c1", "c2", "c9"])
986            .unwrap();
987        assert_eq!(mapped.len(), 2);
988        assert_eq!(mapped["c1"], "m1");
989        assert_eq!(mapped["c2"], "m2");
990        assert!(
991            cache
992                .get_cc_mapped_hashes("maps-b", &["c1"])
993                .unwrap()
994                .is_empty()
995        );
996        let scans = cache.get_cc_asm_scans(&["c1", "c3", "c9"]).unwrap();
997        assert_eq!(scans.len(), 2);
998        assert_eq!(scans["c1"], "");
999        assert_eq!(scans["c3"], ".incbin");
1000        cache
1001            .put_cc_mapped_hashes("maps-a", &[("c1".to_string(), "changed".to_string())])
1002            .unwrap();
1003        cache
1004            .put_cc_asm_scans(&[("c3".to_string(), String::new())])
1005            .unwrap();
1006        assert_eq!(
1007            cache.get_cc_mapped_hashes("maps-a", &["c1"]).unwrap()["c1"],
1008            "m1"
1009        );
1010        assert_eq!(cache.get_cc_asm_scans(&["c3"]).unwrap()["c3"], ".incbin");
1011    }
1012
1013    /// A lookup larger than one statement's chunk still finds every row,
1014    /// and an empty lookup issues nothing.
1015    #[test]
1016    fn cc_memo_lookups_span_chunks() {
1017        let dir = tempfile::tempdir().unwrap();
1018        let path = dir.path().join("index.db");
1019        let cache = FileHashCache::open(&path).unwrap();
1020        let n = MEMO_LOOKUP_CHUNK * 2 + 3;
1021        let pairs: Vec<(String, String)> = (0..n)
1022            .map(|i| (format!("content-{i}"), format!("mapped-{i}")))
1023            .collect();
1024        cache.put_cc_mapped_hashes("maps", &pairs).unwrap();
1025        cache.put_cc_asm_scans(&pairs).unwrap();
1026        let mut wanted: Vec<&str> = pairs.iter().map(|(c, _)| c.as_str()).collect();
1027        wanted.push("content-missing");
1028        let mapped = cache.get_cc_mapped_hashes("maps", &wanted).unwrap();
1029        assert_eq!(mapped.len(), n);
1030        assert_eq!(
1031            mapped[&format!("content-{}", n - 1)],
1032            format!("mapped-{}", n - 1)
1033        );
1034        assert_eq!(mapped["content-0"], "mapped-0");
1035        let scans = cache.get_cc_asm_scans(&wanted).unwrap();
1036        assert_eq!(scans.len(), n);
1037        assert_eq!(
1038            scans[&format!("content-{}", MEMO_LOOKUP_CHUNK)],
1039            format!("mapped-{}", MEMO_LOOKUP_CHUNK)
1040        );
1041        assert!(cache.get_cc_mapped_hashes("maps", &[]).unwrap().is_empty());
1042        assert!(cache.get_cc_asm_scans(&[]).unwrap().is_empty());
1043    }
1044
1045    #[test]
1046    fn cc_memo_pruning_preserves_shared_live_inputs_and_daily_hits() {
1047        let dir = tempfile::tempdir().unwrap();
1048        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1049        cache
1050            .put_cc_preprocess_memo_inputs(
1051                "old",
1052                "a",
1053                &[input("shared.h", "s"), input("orphan.h", "o")],
1054            )
1055            .unwrap();
1056        cache
1057            .put_cc_preprocess_memo_inputs("live", "b", &[input("shared.h", "s")])
1058            .unwrap();
1059        cache
1060            .db()
1061            .execute(
1062                "UPDATE cc_preprocess_memos SET last_used = unixepoch() - 2678400",
1063                [],
1064            )
1065            .unwrap();
1066        assert!(
1067            cache
1068                .get_cc_preprocess_memo("live")
1069                .unwrap()
1070                .unwrap()
1071                .needs_touch
1072        );
1073        assert_eq!(cache.touch_cc_preprocess_memo("live").unwrap(), 1);
1074        assert_eq!(cache.touch_cc_preprocess_memo("live").unwrap(), 0);
1075        assert!(
1076            !cache
1077                .get_cc_preprocess_memo("live")
1078                .unwrap()
1079                .unwrap()
1080                .needs_touch
1081        );
1082        assert_eq!(cache.prune_cc_preprocess_memos().unwrap(), (1, 1));
1083        assert!(cache.get_cc_preprocess_memo("old").unwrap().is_none());
1084        assert_eq!(
1085            cache
1086                .get_cc_preprocess_memo("live")
1087                .unwrap()
1088                .unwrap()
1089                .inputs
1090                .len(),
1091            1
1092        );
1093        assert_eq!(count(&cache, "cc_memo_input_refs"), 1);
1094        assert_eq!(cache.prune_cc_preprocess_memos().unwrap(), (0, 0));
1095    }
1096
1097    #[test]
1098    fn cc_memo_missing_references_fail_closed() {
1099        let dir = tempfile::tempdir().unwrap();
1100        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1101        cache
1102            .put_cc_preprocess_memo_inputs("unit", "h", &[input("a.h", "a"), input("b.h", "b")])
1103            .unwrap();
1104        cache.db().execute("DELETE FROM cc_memo_input_refs WHERE input_id = (SELECT min(id) FROM cc_memo_inputs)", []).unwrap();
1105        assert!(cache.get_cc_preprocess_memo("unit").unwrap().is_none());
1106        cache
1107            .db()
1108            .execute_batch("PRAGMA foreign_keys = OFF")
1109            .unwrap();
1110        cache
1111            .db()
1112            .execute("DELETE FROM cc_memo_inputs", [])
1113            .unwrap();
1114        assert!(cache.get_cc_preprocess_memo("unit").unwrap().is_none());
1115    }
1116
1117    #[test]
1118    fn cc_memo_legacy_migration_keeps_other_tables_and_is_idempotent() {
1119        let dir = tempfile::tempdir().unwrap();
1120        let path = dir.path().join("index.db");
1121        let db = Connection::open(&path).unwrap();
1122        db.execute_batch("CREATE TABLE cc_preprocess_memos(memo_key TEXT PRIMARY KEY, preprocessed_hash TEXT, inputs_json TEXT);
1123            INSERT INTO cc_preprocess_memos VALUES ('old', 'digest', '[{}]');
1124            CREATE TABLE unrelated(value TEXT); INSERT INTO unrelated VALUES ('keep');").unwrap();
1125        drop(db);
1126        let cache = FileHashCache::open(&path).unwrap();
1127        assert!(cache.get_cc_preprocess_memo("old").unwrap().is_none());
1128        assert_eq!(
1129            cache
1130                .db()
1131                .query_row("SELECT value FROM unrelated", [], |row| row
1132                    .get::<_, String>(0))
1133                .unwrap(),
1134            "keep"
1135        );
1136        cache
1137            .put_cc_preprocess_memo_inputs("new", "hash", &[input("a.h", "a")])
1138            .unwrap();
1139        ensure_schema(cache.db()).unwrap();
1140        assert!(cache.get_cc_preprocess_memo("new").unwrap().is_some());
1141        assert_eq!(cache.db().query_row("SELECT count(*) FROM pragma_table_info('cc_preprocess_memos') WHERE name = 'inputs_json'", [], |row| row.get::<_, i64>(0)).unwrap(), 0);
1142    }
1143
1144    #[test]
1145    fn cc_memo_repeated_headers_have_bounded_storage() {
1146        let dir = tempfile::tempdir().unwrap();
1147        let path = dir.path().join("index.db");
1148        let cache = FileHashCache::open(&path).unwrap();
1149        let inputs: Vec<_> = (0..100)
1150            .map(|i| {
1151                input(
1152                    &format!("include/{i:04}-{}.h", "x".repeat(100)),
1153                    &"a".repeat(64),
1154                )
1155            })
1156            .collect();
1157        let old_payload_bytes = serde_json::to_vec(&inputs).unwrap().len() * 200;
1158        for i in 0..200 {
1159            cache
1160                .put_cc_preprocess_memo_inputs(&format!("{i:064}"), "hash", &inputs)
1161                .unwrap();
1162        }
1163        assert_eq!(count(&cache, "cc_memo_inputs"), 100);
1164        assert_eq!(count(&cache, "cc_memo_input_refs"), 20_000);
1165        let pages: u64 = cache
1166            .db()
1167            .pragma_query_value(None, "page_count", |r| r.get::<_, u32>(0).map(u64::from))
1168            .unwrap();
1169        let size: u64 = cache
1170            .db()
1171            .pragma_query_value(None, "page_size", |r| r.get::<_, u32>(0).map(u64::from))
1172            .unwrap();
1173        assert!(
1174            pages * size < old_payload_bytes as u64 / 5,
1175            "{} bytes vs {} bytes of legacy payloads",
1176            pages * size,
1177            old_payload_bytes
1178        );
1179    }
1180
1181    #[test]
1182    fn cc_memo_failed_replacement_rolls_back_the_whole_record() {
1183        let dir = tempfile::tempdir().unwrap();
1184        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1185        cache
1186            .put_cc_preprocess_memo_inputs("unit", "old", &[input("old.h", "old")])
1187            .unwrap();
1188        cache
1189            .db()
1190            .execute_batch(
1191                "CREATE TRIGGER reject_reference BEFORE INSERT ON cc_memo_input_refs
1192            BEGIN SELECT RAISE(ABORT, 'injected failure'); END;",
1193            )
1194            .unwrap();
1195        assert!(
1196            cache
1197                .put_cc_preprocess_memo_inputs("unit", "new", &[input("new.h", "new")])
1198                .is_err()
1199        );
1200        let memo = cache.get_cc_preprocess_memo("unit").unwrap().unwrap();
1201        assert_eq!(memo.preprocessed_hash, "old");
1202        assert_eq!(memo.inputs, vec![read_back(input("old.h", "old"))]);
1203        assert_eq!(count(&cache, "cc_memo_inputs"), 1);
1204        assert_eq!(count(&cache, "cc_memo_input_refs"), 1);
1205    }
1206
1207    #[test]
1208    fn cc_memo_concurrent_reads_never_mix_digest_and_inputs() {
1209        let dir = tempfile::tempdir().unwrap();
1210        let path = dir.path().join("index.db");
1211        let cache = FileHashCache::open(&path).unwrap();
1212        cache
1213            .put_cc_preprocess_memo_inputs("unit", "a", &[input("a.h", "a")])
1214            .unwrap();
1215        let writer = std::thread::spawn(move || {
1216            let cache = FileHashCache::open(&path).unwrap();
1217            for i in 0..300 {
1218                let value = if i % 2 == 0 { "a" } else { "b" };
1219                cache
1220                    .put_cc_preprocess_memo_inputs(
1221                        "unit",
1222                        value,
1223                        &[input(&format!("{value}.h"), value)],
1224                    )
1225                    .unwrap();
1226            }
1227        });
1228        for _ in 0..600 {
1229            let memo = cache.get_cc_preprocess_memo("unit").unwrap().unwrap();
1230            assert_eq!(memo.inputs.len(), 1);
1231            assert_eq!(memo.preprocessed_hash, memo.inputs[0].content);
1232        }
1233        writer.join().unwrap();
1234    }
1235
1236    #[test]
1237    #[cfg(unix)]
1238    fn sparse_index_repair_reclaims_legacy_pages_and_preserves_records() {
1239        let dir = tempfile::tempdir().unwrap();
1240        let path = dir.path().join("index.db");
1241        let db = Connection::open(&path).unwrap();
1242        db.execute_batch(
1243            "CREATE TABLE cc_preprocess_memos(memo_key TEXT PRIMARY KEY, inputs_json TEXT);
1244            INSERT INTO cc_preprocess_memos VALUES ('legacy', zeroblob(83886080));",
1245        )
1246        .unwrap();
1247        drop(db);
1248        let cache = FileHashCache::open(&path).unwrap();
1249        cache
1250            .put_cc_preprocess_memo_inputs("live", "digest", &[input("live.h", "live")])
1251            .unwrap();
1252        let (before, after) = cache
1253            .compact_sparse_index()
1254            .unwrap()
1255            .expect("large freelist must compact");
1256        assert!(before >= 80 << 20);
1257        assert!(after < before / 4);
1258        assert_eq!(
1259            cache
1260                .get_cc_preprocess_memo("live")
1261                .unwrap()
1262                .unwrap()
1263                .preprocessed_hash,
1264            "digest"
1265        );
1266        assert_eq!(cache.compact_sparse_index().unwrap(), None);
1267    }
1268
1269    #[test]
1270    fn a_memo_input_names_its_local_path() {
1271        let first = input("shared.h", "one");
1272        assert_eq!(first.local_path(), "/checkout/shared.h");
1273    }
1274
1275    #[test]
1276    fn a_memo_without_inputs_is_not_a_memo() {
1277        let dir = tempfile::tempdir().unwrap();
1278        let cache = FileHashCache::open(&dir.path().join("index.db")).unwrap();
1279        cache
1280            .put_cc_preprocess_memo_inputs("empty", "hash-e", &[])
1281            .unwrap();
1282        assert!(
1283            cache.get_cc_preprocess_memo("empty").unwrap().is_none(),
1284            "a preprocess always reads at least the source"
1285        );
1286        let json = serde_json::to_string(&[input("a.h", "one")]).unwrap();
1287        cache
1288            .put_cc_preprocess_memo("json", "hash-j", &json)
1289            .unwrap();
1290        let memo = cache.get_cc_preprocess_memo("json").unwrap().unwrap();
1291        assert_eq!(memo.preprocessed_hash, "hash-j");
1292        assert_eq!(memo.inputs, vec![read_back(input("a.h", "one"))]);
1293        assert!(
1294            cache
1295                .put_cc_preprocess_memo("bad", "hash-b", "not json")
1296                .is_err()
1297        );
1298    }
1299}
kache_store/cc_memo.rs

kache_store/
cc_memo.rs
