1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use super::ObjectStore;
use crate::checksum;
use crate::config::RestoreMethod;
/// Distinguishes concurrent temp files within one process; combined with the
/// pid so concurrent rsconstruct invocations never collide either.
pub(super) static NEXT_TMP_ID: AtomicU64 = AtomicU64::new(0);
impl ObjectStore {
/// Calculate SHA-256 checksum of bytes
pub fn calculate_checksum_bytes(data: &[u8]) -> String {
checksum::bytes_checksum(data)
}
/// Get object path for a checksum (e.g., .rsconstruct/objects/ab/cdef123...)
pub(super) fn object_path(&self, checksum: &str) -> PathBuf {
let (prefix, rest) = super::shard(checksum);
self.objects_dir.join(prefix).join(rest)
}
/// Path of the zstd-compressed variant of an object. The `.zst` suffix
/// records the storage format in the filename, so readers key off the
/// object's actual format rather than the current `compression` setting —
/// toggling the setting must never misread existing objects.
pub(super) fn compressed_object_path(&self, checksum: &str) -> PathBuf {
let mut path = self.object_path(checksum).into_os_string();
path.push(".zst");
PathBuf::from(path)
}
/// On-disk size of a stored object, whichever format it was written in.
/// Callers must not assume `object_path` — under `compression = true` the
/// object only exists at `compressed_object_path`, and stat'ing the plain
/// path silently reports zero.
pub(super) fn object_size(&self, checksum: &str) -> u64 {
for path in [
self.object_path(checksum),
self.compressed_object_path(checksum),
] {
if let Ok(metadata) = fs::metadata(&path) {
return metadata.len();
}
}
0
}
/// Store content in object store, returns checksum.
/// The checksum is always computed on the **original** (uncompressed) content
/// so cache keys remain stable regardless of compression setting.
/// Objects are made read-only to prevent accidental modification via hardlinks.
///
/// The blob is written to a unique temp file and renamed into place: a
/// crash can never leave a truncated object at the content-addressed path,
/// and concurrent stores of identical content don't collide.
pub(super) fn store_object(&self, content: &[u8]) -> Result<String> {
let checksum = Self::calculate_checksum_bytes(content);
// Short-circuit only when the object exists in the CURRENT format:
// a format-blind `has_object` check made toggling `compression = true`
// a silent no-op for every object already stored uncompressed.
let object_path = if self.compression {
self.compressed_object_path(&checksum)
} else {
self.object_path(&checksum)
};
if object_path.exists() {
return Ok(checksum);
}
let parent = object_path
.parent()
.context("Object path has no parent directory")?;
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create object directory: {}", parent.display()))?;
let blob = if self.compression {
zstd::encode_all(content, 0).context("Failed to zstd-compress object")?
} else {
content.to_vec()
};
let tmp_path = parent.join(format!(
".tmp-{}-{}",
std::process::id(),
NEXT_TMP_ID.fetch_add(1, Ordering::Relaxed)
));
fs::write(&tmp_path, &blob)
.with_context(|| format!("Failed to write object temp file: {}", tmp_path.display()))?;
let mut perms = fs::metadata(&tmp_path)
.with_context(|| {
format!(
"Failed to read object temp file metadata: {}",
tmp_path.display()
)
})?
.permissions();
perms.set_readonly(true);
fs::set_permissions(&tmp_path, perms).with_context(|| {
format!(
"Failed to set object temp file read-only: {}",
tmp_path.display()
)
})?;
if let Err(e) = fs::rename(&tmp_path, &object_path) {
// A concurrent store may have placed the identical object first;
// that's success. Otherwise clean up the temp file and fail.
let mut writable = fs::metadata(&tmp_path).map(|m| m.permissions());
if let Ok(ref mut perms) = writable {
perms.set_readonly(false);
let _ = fs::set_permissions(&tmp_path, perms.clone());
}
let _ = fs::remove_file(&tmp_path);
if !object_path.exists() {
return Err(e).with_context(|| {
format!(
"Failed to move object into place: {}",
object_path.display()
)
});
}
}
Ok(checksum)
}
/// Check if an object exists in the store, in either storage format.
pub(super) fn has_object(&self, checksum: &str) -> bool {
self.object_path(checksum).exists() || self.compressed_object_path(checksum).exists()
}
/// Remove an object from the store, in whichever format it is held.
/// Objects are read-only; removing one needs only its directory to be
/// writable, so no mode is changed.
pub(super) fn remove_object(&self, checksum: &str) -> Result<()> {
for path in [
self.object_path(checksum),
self.compressed_object_path(checksum),
] {
match fs::remove_file(&path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => {
return Err(e).with_context(|| {
format!("Failed to remove cache object: {}", path.display())
});
}
}
}
Ok(())
}
/// Restore a file from the object store, applying the stored file mode.
///
/// The restore path is chosen by the object's actual on-disk format, not
/// the current `compression` setting — a compressed object is always
/// decompressed into a fresh file. A hardlinked output shares the
/// read-only object inode, so no mode is ever applied through a hardlink —
/// chmodding the link would make the cache object itself writable and any
/// later in-place write would silently corrupt it. Files whose stored mode
/// needs exec bits are restored by copy so the mode can be applied
/// faithfully.
pub(super) fn restore_file(
&self,
checksum: &str,
output_path: &Path,
mode: Option<u32>,
) -> Result<()> {
let object_path = self.object_path(checksum);
if !object_path.exists() {
// Only the compressed variant exists: decompress into the output.
let content = self
.read_object(checksum)
.with_context(|| format!("Failed to read cached object: {checksum}"))?;
fs::write(output_path, &content).with_context(|| {
format!(
"Failed to write decompressed output: {}",
output_path.display()
)
})?;
crate::platform::set_permissions_mode(output_path, mode.unwrap_or(0o644))
.with_context(|| {
format!(
"Failed to set permissions on restored file: {}",
output_path.display()
)
})?;
return Ok(());
}
// A hardlink shares the object's inode and so its mode, 0444. That is
// only right when 0444 grants nothing the recorded mode withheld:
// an executable needs its exec bits, and a file the tool wrote
// private (0600, 0640) must not come back readable by group or
// others. Both are restored by copy, with the recorded mode.
let needs_own_mode = mode.is_some_and(|m| m & 0o111 != 0 || 0o044 & !m != 0);
match self.restore_method {
RestoreMethod::Hardlink if !needs_own_mode => {
fs::hard_link(&object_path, output_path)
.with_context(|| format!("Failed to hard link from cache: {checksum}. If on a cross-filesystem setup, set restore_method = \"copy\" in rsconstruct.toml."))?;
}
RestoreMethod::Hardlink | RestoreMethod::Copy => {
fs::copy(&object_path, output_path)
.with_context(|| format!("Failed to copy from cache: {checksum}"))?;
crate::platform::set_permissions_mode(output_path, mode.unwrap_or(0o644))
.with_context(|| {
format!(
"Failed to set permissions on restored file: {}",
output_path.display()
)
})?;
}
RestoreMethod::Auto => unreachable!("Auto should be resolved before use"),
}
Ok(())
}
/// Read an object's original content, decompressing if the object is
/// stored compressed. The format is detected from which variant exists on
/// disk, so objects written under either `compression` setting stay
/// readable.
pub(crate) fn read_object(&self, checksum: &str) -> Result<Vec<u8>> {
let plain_path = self.object_path(checksum);
if plain_path.exists() {
return fs::read(&plain_path)
.with_context(|| format!("Failed to read object: {checksum}"));
}
let compressed_path = self.compressed_object_path(checksum);
let raw = fs::read(&compressed_path)
.with_context(|| format!("Failed to read object: {checksum}"))?;
zstd::decode_all(raw.as_slice())
.with_context(|| format!("Failed to decompress object: {checksum}"))
}
/// Convert path to string for storage. Paths are already relative.
pub(super) fn path_string(path: &Path) -> String {
path.display().to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn store_with_db(dir: &Path, db_name: &str) -> ObjectStore {
ObjectStore::new_at(dir, db_name)
}
fn store_in(dir: &Path) -> ObjectStore {
ObjectStore::new_in(dir)
}
/// A hardlink restore shares the object's inode: applying the stored mode
/// through the link would make the cache object writable and let later
/// in-place writes silently corrupt it.
#[test]
fn hardlink_restore_keeps_object_read_only() {
let tmp = tempfile::TempDir::new().unwrap();
let store = store_in(tmp.path());
let checksum = store.store_object(b"cached content").unwrap();
let out = tmp.path().join("out.txt");
store.restore_file(&checksum, &out, Some(0o644)).unwrap();
let obj_meta = fs::metadata(store.object_path(&checksum)).unwrap();
assert!(
obj_meta.permissions().readonly(),
"cache object must stay read-only after a hardlink restore"
);
let mode = crate::platform::get_mode(&fs::metadata(&out).unwrap());
assert_eq!(
mode & 0o222,
0,
"hardlink-restored output shares the object inode and must stay read-only"
);
}
/// A file the tool wrote private comes back private. A hardlink shares
/// the object's 0444, which made a restored 0600 output readable by
/// everyone; such a file is restored by copy with its recorded mode.
/// The common modes (0644, 0664) still restore as hardlinks.
#[test]
fn restore_does_not_widen_a_private_mode() {
let tmp = tempfile::TempDir::new().unwrap();
let store = store_in(tmp.path());
let checksum = store.store_object(b"secret").unwrap();
let ino = |p: &Path| std::os::unix::fs::MetadataExt::ino(&fs::metadata(p).unwrap());
let object = store.object_path(&checksum);
for mode in [0o600, 0o640] {
let out = tmp.path().join(format!("private-{mode:o}"));
store.restore_file(&checksum, &out, Some(mode)).unwrap();
let restored = crate::platform::get_mode(&fs::metadata(&out).unwrap()) & 0o777;
assert_eq!(restored, mode, "restored {restored:o}, recorded {mode:o}");
assert_ne!(ino(&out), ino(&object), "must be a copy, not a link");
}
for mode in [0o644, 0o664] {
let out = tmp.path().join(format!("public-{mode:o}"));
store.restore_file(&checksum, &out, Some(mode)).unwrap();
assert_eq!(ino(&out), ino(&object), "{mode:o} still hardlinks");
}
}
/// Objects written under one `compression` setting must stay readable and
/// restorable after the setting is toggled: the storage format lives in
/// the object filename, not in the config.
#[test]
fn compression_toggle_roundtrip() {
let tmp = tempfile::TempDir::new().unwrap();
let compressed_store = ObjectStore {
compression: true,
..store_with_db(tmp.path(), "db1.redb")
};
let checksum = compressed_store
.store_object(b"compressed at store time")
.unwrap();
assert!(compressed_store.compressed_object_path(&checksum).exists());
// Same on-disk store, compression now off
let plain_store = store_with_db(tmp.path(), "db2.redb");
assert!(
plain_store.has_object(&checksum),
"toggled store must still see the object"
);
assert_eq!(
plain_store.read_object(&checksum).unwrap(),
b"compressed at store time",
"read must decompress based on the object's actual format"
);
let out = tmp.path().join("restored.txt");
plain_store
.restore_file(&checksum, &out, Some(0o644))
.unwrap();
assert_eq!(
fs::read(&out).unwrap(),
b"compressed at store time",
"restore must never emit raw zstd bytes as file content"
);
// And the reverse direction: plain object read with compression on
let checksum2 = plain_store.store_object(b"plain at store time").unwrap();
assert_eq!(
compressed_store.read_object(&checksum2).unwrap(),
b"plain at store time"
);
}
/// Executable outputs cannot be restored by hardlink (the mode would have
/// to be applied to the shared inode) — they must fall back to copy.
#[test]
fn exec_mode_restores_via_copy() {
let tmp = tempfile::TempDir::new().unwrap();
let store = store_in(tmp.path());
let checksum = store.store_object(b"#!/bin/sh\n").unwrap();
let out = tmp.path().join("script.sh");
store.restore_file(&checksum, &out, Some(0o755)).unwrap();
assert!(
fs::metadata(store.object_path(&checksum))
.unwrap()
.permissions()
.readonly(),
"cache object must stay read-only after an exec-mode restore"
);
let mode = crate::platform::get_mode(&fs::metadata(&out).unwrap());
assert_ne!(mode & 0o111, 0, "restored script must be executable");
assert_ne!(mode & 0o200, 0, "copy-restored file must be writable");
}
}