zccache/depgraph/snapshot/
mod.rs

1//! Disk persistence for the dependency graph via rkyv zero-copy serialization.
2//!
3//! Saves/loads the graph to `~/.zccache/depgraph/depgraph.bin` so warm contexts
4//! survive daemon restarts and cache hits resume immediately.
5//!
6//! Split into focused submodules so each file stays under 1,000 LOC:
7//! - this file: snapshot types, error type, [`DepGraph::to_snapshot`] /
8//!   [`DepGraph::from_snapshot`] conversion methods, and the tiny
9//!   `paths_to_strings` / `strings_to_paths` helpers used by both
10//!   conversion and tests.
11//! - `persistence`: file I/O — [`save_to_file`], [`load_from_file`],
12//!   [`classify_load`], [`depgraph_file_path`], and [`DepGraphLoadOutcome`].
13//! - `tests` (cfg(test) only): split per concern — roundtrip, persistence,
14//!   behavioral.
15
16use std::path::Path;
17use std::time::{Instant, SystemTime, UNIX_EPOCH};
18
19use crate::core::NormalizedPath;
20use crate::hash::ContentHash;
21use dashmap::DashMap;
22use rayon::prelude::*;
23use rkyv::{Archive, Deserialize, Serialize};
24
25use super::context::{ArtifactKey, CompileContext, ContextKey};
26use super::graph::{ContextEntry, ContextState, DepGraph, FileEntry};
27use super::scanner::{IncludeDirective, IncludeKind};
28use super::search_paths::IncludeSearchPaths;
29
30mod persistence;
31#[cfg(test)]
32mod tests;
33
34pub use persistence::{
35    classify_load, depgraph_file_path, load_from_file, save_to_file, DepGraphLoadOutcome,
36};
37
38/// On-disk format version. Bump when snapshot layout changes.
39pub const DEPGRAPH_VERSION: u32 = 5;
40
41/// Magic bytes identifying a depgraph snapshot file ("ZCDG").
42pub const DEPGRAPH_MAGIC: [u8; 4] = [0x5A, 0x43, 0x44, 0x47];
43
44/// Header size: 4 (magic) + 4 (version) + 8 (payload len) = 16 bytes.
45pub(crate) const HEADER_SIZE: usize = 16;
46
47// ---------------------------------------------------------------------------
48// Snapshot types (rkyv-serializable mirrors of the in-memory types)
49// ---------------------------------------------------------------------------
50
51#[derive(Archive, Serialize, Deserialize)]
52#[archive(check_bytes)]
53pub struct DepGraphSnapshot {
54    pub files: Vec<FileEntrySnapshot>,
55    pub contexts: Vec<ContextEntrySnapshot>,
56    pub stats: SnapshotStats,
57}
58
59#[derive(Archive, Serialize, Deserialize)]
60#[archive(check_bytes)]
61pub struct FileEntrySnapshot {
62    pub path: String,
63    pub includes: Vec<IncludeDirectiveSnapshot>,
64}
65
66#[derive(Archive, Serialize, Deserialize)]
67#[archive(check_bytes)]
68pub struct IncludeDirectiveSnapshot {
69    /// 0=Quoted, 1=AngleBracket, 2=Computed
70    pub kind: u8,
71    pub path: String,
72    pub line: u32,
73}
74
75#[derive(Archive, Serialize, Deserialize)]
76#[archive(check_bytes)]
77pub struct ContextEntrySnapshot {
78    pub context_key: [u8; 32],
79    pub key_root: Option<String>,
80    pub source_file: String,
81    pub iquote: Vec<String>,
82    pub user: Vec<String>,
83    pub system: Vec<String>,
84    pub after: Vec<String>,
85    pub defines: Vec<String>,
86    pub flags: Vec<String>,
87    pub force_includes: Vec<String>,
88    pub unknown_flags: Vec<String>,
89    pub resolved_includes: Vec<String>,
90    pub unresolved_includes: Vec<String>,
91    pub has_computed_includes: bool,
92    pub artifact_key: Option<[u8; 32]>,
93    pub last_file_hashes: Vec<(String, [u8; 32])>,
94    pub rustc_externs: Vec<RustcExternSnapshot>,
95    /// 0=Cold, 1=Warm, 2=Stale
96    pub state: u8,
97}
98
99#[derive(Archive, Serialize, Deserialize)]
100#[archive(check_bytes)]
101pub struct RustcExternSnapshot {
102    pub name: String,
103    pub path: String,
104}
105
106#[derive(Archive, Serialize, Deserialize)]
107#[archive(check_bytes)]
108pub struct SnapshotStats {
109    pub saved_at_epoch_secs: u64,
110    pub file_count: u64,
111    pub context_count: u64,
112}
113
114// ---------------------------------------------------------------------------
115// Error type
116// ---------------------------------------------------------------------------
117
118#[derive(Debug, thiserror::Error)]
119pub enum SnapshotError {
120    #[error("io error: {0}")]
121    Io(#[from] std::io::Error),
122
123    #[error("bad magic bytes in depgraph file")]
124    BadMagic,
125
126    #[error("depgraph version mismatch: file has v{file}, expected v{expected}")]
127    VersionMismatch { file: u32, expected: u32 },
128
129    #[error("corrupt depgraph file: {0}")]
130    Corrupt(String),
131}
132
133// ---------------------------------------------------------------------------
134// Conversion: DepGraph <-> Snapshot
135// ---------------------------------------------------------------------------
136
137impl DepGraph {
138    /// Create a serializable snapshot of the current graph state.
139    pub fn to_snapshot(&self) -> DepGraphSnapshot {
140        let files: Vec<FileEntrySnapshot> = self
141            .files_iter()
142            .map(|entry| {
143                let path = entry.key().to_string_lossy().into_owned();
144                let includes = entry
145                    .value()
146                    .includes
147                    .iter()
148                    .map(|d| IncludeDirectiveSnapshot {
149                        kind: match &d.kind {
150                            IncludeKind::Quoted => 0,
151                            IncludeKind::AngleBracket => 1,
152                            IncludeKind::Computed(_) => 2,
153                        },
154                        path: d.path.clone(),
155                        line: d.line,
156                    })
157                    .collect();
158                FileEntrySnapshot { path, includes }
159            })
160            .collect();
161
162        let contexts: Vec<ContextEntrySnapshot> = self
163            .contexts_iter()
164            .map(|entry| {
165                let key = entry.key();
166                let ctx = entry.value();
167                let rustc_externs = self
168                    .get_rustc_externs(key)
169                    .unwrap_or_default()
170                    .into_iter()
171                    .map(|(name, path)| RustcExternSnapshot {
172                        name,
173                        path: path.to_string_lossy().into_owned(),
174                    })
175                    .collect();
176                ContextEntrySnapshot {
177                    context_key: *key.hash().as_bytes(),
178                    key_root: ctx
179                        .key_root
180                        .as_ref()
181                        .map(|p| p.to_string_lossy().into_owned()),
182                    source_file: ctx.context.source_file.to_string_lossy().into_owned(),
183                    iquote: paths_to_strings(&ctx.context.include_search.iquote),
184                    user: paths_to_strings(&ctx.context.include_search.user),
185                    system: paths_to_strings(&ctx.context.include_search.system),
186                    after: paths_to_strings(&ctx.context.include_search.after),
187                    defines: ctx.context.defines.clone(),
188                    flags: ctx.context.flags.clone(),
189                    force_includes: paths_to_strings(&ctx.context.force_includes),
190                    unknown_flags: ctx.context.unknown_flags.clone(),
191                    resolved_includes: paths_to_strings(&ctx.resolved_includes),
192                    unresolved_includes: ctx.unresolved_includes.clone(),
193                    has_computed_includes: ctx.has_computed_includes,
194                    artifact_key: ctx.artifact_key.map(|k| *k.hash().as_bytes()),
195                    last_file_hashes: ctx
196                        .last_file_hashes
197                        .iter()
198                        .map(|(p, h)| (p.to_string_lossy().into_owned(), *h.as_bytes()))
199                        .collect(),
200                    rustc_externs,
201                    state: match ctx.state {
202                        ContextState::Cold => 0,
203                        ContextState::Warm => 1,
204                        ContextState::Stale => 2,
205                    },
206                }
207            })
208            .collect();
209
210        DepGraphSnapshot {
211            stats: SnapshotStats {
212                saved_at_epoch_secs: SystemTime::now()
213                    .duration_since(UNIX_EPOCH)
214                    .unwrap_or_default()
215                    .as_secs(),
216                file_count: files.len() as u64,
217                context_count: contexts.len() as u64,
218            },
219            files,
220            contexts,
221        }
222    }
223
224    /// Reconstruct a `DepGraph` from a deserialized snapshot.
225    pub fn from_snapshot(snap: DepGraphSnapshot) -> Self {
226        let files: DashMap<NormalizedPath, FileEntry> = DashMap::new();
227        snap.files.into_par_iter().for_each(|f| {
228            let path = NormalizedPath::from(f.path.as_str());
229            let includes = f
230                .includes
231                .into_iter()
232                .map(|d| {
233                    let kind = match d.kind {
234                        0 => IncludeKind::Quoted,
235                        1 => IncludeKind::AngleBracket,
236                        _ => IncludeKind::Computed(d.path.clone()),
237                    };
238                    IncludeDirective {
239                        kind,
240                        path: d.path,
241                        line: d.line,
242                    }
243                })
244                .collect();
245            files.insert(
246                path,
247                FileEntry {
248                    includes,
249                    scanned_at: Instant::now(),
250                },
251            );
252        });
253
254        let contexts: DashMap<ContextKey, ContextEntry> = DashMap::new();
255        let rustc_externs: DashMap<ContextKey, Vec<(String, NormalizedPath)>> = DashMap::new();
256        snap.contexts.into_par_iter().for_each(|c| {
257            let key = ContextKey::from_raw(c.context_key);
258            let externs: Vec<(String, NormalizedPath)> = c
259                .rustc_externs
260                .into_iter()
261                .map(|entry| (entry.name, NormalizedPath::from(entry.path.as_str())))
262                .collect();
263            let context = CompileContext {
264                source_file: NormalizedPath::from(c.source_file.as_str()),
265                include_search: IncludeSearchPaths {
266                    iquote: strings_to_paths(c.iquote),
267                    user: strings_to_paths(c.user),
268                    system: strings_to_paths(c.system),
269                    after: strings_to_paths(c.after),
270                },
271                defines: c.defines,
272                flags: c.flags,
273                force_includes: strings_to_paths(c.force_includes),
274                unknown_flags: c.unknown_flags,
275            };
276            let entry = ContextEntry {
277                context,
278                key_root: c.key_root.map(|root| NormalizedPath::from(root.as_str())),
279                resolved_includes: strings_to_paths(c.resolved_includes),
280                unresolved_includes: c.unresolved_includes,
281                has_computed_includes: c.has_computed_includes,
282                artifact_key: c.artifact_key.map(ArtifactKey::from_raw),
283                last_file_hashes: c
284                    .last_file_hashes
285                    .into_iter()
286                    .map(|(p, h)| (NormalizedPath::from(p.as_str()), ContentHash::from_bytes(h)))
287                    .collect(),
288                last_accessed: Instant::now(),
289                state: match c.state {
290                    0 => ContextState::Cold,
291                    1 => ContextState::Warm,
292                    _ => ContextState::Stale,
293                },
294            };
295            contexts.insert(key, entry);
296            if !externs.is_empty() {
297                rustc_externs.insert(key, externs);
298            }
299        });
300
301        DepGraph::from_maps_with_rustc_externs(files, contexts, rustc_externs)
302    }
303}
304
305pub(crate) fn paths_to_strings<P: AsRef<Path>>(paths: &[P]) -> Vec<String> {
306    paths
307        .iter()
308        .map(|p| p.as_ref().to_string_lossy().into_owned())
309        .collect()
310}
311
312pub(crate) fn strings_to_paths(strings: Vec<String>) -> Vec<NormalizedPath> {
313    strings.into_iter().map(NormalizedPath::from).collect()
314}
zccache/depgraph/snapshot/mod.rs

zccache/depgraph/snapshot/
mod.rs
