1use crate::data::{DataCatalogError, DistributionSource, ProductIdentity};
13use serde::{Deserialize, Serialize};
14use sha2::{Digest, Sha256};
15use std::fmt::Write as _;
16
17pub const EXACT_CACHE_SCHEMA_VERSION: u8 = 3;
19
20pub const EXACT_CACHE_CONTROL_DIRECTORY: &str = ".sidereon-cache-v3";
22
23pub const EXACT_CACHE_MARKER_FILENAME: &str = "current.json";
25
26#[derive(Debug, thiserror::Error)]
28pub enum ExactCacheError {
29 #[error("invalid exact product identity: {0}")]
31 Identity(#[from] DataCatalogError),
32 #[error("invalid exact-cache entry identifier")]
34 InvalidEntryId,
35 #[error("invalid or mismatched exact-cache commit: {0}")]
37 InvalidCommit(&'static str),
38 #[cfg(not(target_arch = "wasm32"))]
40 #[error("exact-cache {operation} failed: {source}")]
41 Io {
42 operation: &'static str,
44 #[source]
46 source: std::io::Error,
47 },
48 #[cfg(not(target_arch = "wasm32"))]
50 #[error("timed out waiting for the exact-cache lock")]
51 LockTimeout,
52 #[cfg(not(target_arch = "wasm32"))]
54 #[error("durable exact-cache publication is unsupported on this platform")]
55 UnsupportedPlatform,
56}
57
58#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct ExactCacheDigests {
61 pub identity_sha256: String,
63 pub distribution_source: String,
65 pub product_sha256: String,
67 pub product_byte_length: u64,
69 pub archive_sha256: String,
71 pub archive_byte_length: u64,
73 pub provenance_sha256: String,
75 pub provenance_byte_length: u64,
77}
78
79#[derive(Debug, Clone, PartialEq, Eq)]
81pub struct VerifiedExactCacheCommit {
82 pub entry_id: String,
84 pub digests: ExactCacheDigests,
86}
87
88#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
89#[serde(deny_unknown_fields)]
90struct CommitRecord {
91 schema_version: u8,
92 entry: String,
93 identity_sha256: String,
94 distribution_source: String,
95 product_sha256: String,
96 product_byte_length: u64,
97 archive_sha256: String,
98 archive_byte_length: u64,
99 provenance_sha256: String,
100 provenance_byte_length: u64,
101}
102
103pub fn identity_sha256(identity: &ProductIdentity) -> Result<String, ExactCacheError> {
105 Ok(sha256_hex(&identity.canonical_bytes()?))
106}
107
108pub fn build_commit_record(
116 identity: &ProductIdentity,
117 source: DistributionSource,
118 entry_id: &str,
119 product: &[u8],
120 archive: &[u8],
121 provenance: &[u8],
122) -> Result<Vec<u8>, ExactCacheError> {
123 validate_entry_id(entry_id)?;
124 let record = CommitRecord {
125 schema_version: EXACT_CACHE_SCHEMA_VERSION,
126 entry: entry_id.to_owned(),
127 identity_sha256: identity_sha256(identity)?,
128 distribution_source: source.code().to_owned(),
129 product_sha256: sha256_hex(product),
130 product_byte_length: byte_length(product)?,
131 archive_sha256: sha256_hex(archive),
132 archive_byte_length: byte_length(archive)?,
133 provenance_sha256: sha256_hex(provenance),
134 provenance_byte_length: byte_length(provenance)?,
135 };
136 serde_json::to_vec(&record).map_err(|_| ExactCacheError::InvalidCommit("serialization"))
137}
138
139pub fn verify_commit_record(
146 identity: &ProductIdentity,
147 source: DistributionSource,
148 marker: &[u8],
149 product: &[u8],
150 archive: &[u8],
151 provenance: &[u8],
152) -> Result<VerifiedExactCacheCommit, ExactCacheError> {
153 let record: CommitRecord = serde_json::from_slice(marker)
154 .map_err(|_| ExactCacheError::InvalidCommit("malformed JSON"))?;
155 if record.schema_version != EXACT_CACHE_SCHEMA_VERSION {
156 return Err(ExactCacheError::InvalidCommit("schema version"));
157 }
158 validate_entry_id(&record.entry)?;
159 let expected_identity = identity_sha256(identity)?;
160 let expected = ExactCacheDigests {
161 identity_sha256: expected_identity,
162 distribution_source: source.code().to_owned(),
163 product_sha256: sha256_hex(product),
164 product_byte_length: byte_length(product)?,
165 archive_sha256: sha256_hex(archive),
166 archive_byte_length: byte_length(archive)?,
167 provenance_sha256: sha256_hex(provenance),
168 provenance_byte_length: byte_length(provenance)?,
169 };
170 let actual = ExactCacheDigests {
171 identity_sha256: record.identity_sha256,
172 distribution_source: record.distribution_source,
173 product_sha256: record.product_sha256,
174 product_byte_length: record.product_byte_length,
175 archive_sha256: record.archive_sha256,
176 archive_byte_length: record.archive_byte_length,
177 provenance_sha256: record.provenance_sha256,
178 provenance_byte_length: record.provenance_byte_length,
179 };
180 if actual != expected {
181 return Err(ExactCacheError::InvalidCommit("identity, source, or bytes"));
182 }
183 Ok(VerifiedExactCacheCommit {
184 entry_id: record.entry,
185 digests: actual,
186 })
187}
188
189fn validate_entry_id(entry_id: &str) -> Result<(), ExactCacheError> {
190 if entry_id.len() == 32
191 && entry_id
192 .bytes()
193 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
194 {
195 Ok(())
196 } else {
197 Err(ExactCacheError::InvalidEntryId)
198 }
199}
200
201fn byte_length(bytes: &[u8]) -> Result<u64, ExactCacheError> {
202 u64::try_from(bytes.len()).map_err(|_| ExactCacheError::InvalidCommit("byte length overflow"))
203}
204
205fn sha256_hex(bytes: &[u8]) -> String {
206 let digest = Sha256::digest(bytes);
207 let mut encoded = String::with_capacity(64);
208 for byte in digest {
209 write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail");
210 }
211 encoded
212}
213
214#[cfg(not(target_arch = "wasm32"))]
215mod native {
216 use super::*;
217 use fs2::FileExt;
218 use std::fs::{self, File, OpenOptions};
219 use std::io::{ErrorKind, Write};
220 use std::path::{Path, PathBuf};
221 use std::thread;
222 use std::time::{Duration, Instant};
223
224 const LOCK_FILENAME: &str = ".sidereon-cache.lock";
225
226 #[derive(Debug, Clone)]
228 pub struct CommittedExactCacheEntry {
229 pub entry_id: String,
231 pub product_path: PathBuf,
233 pub archive_path: PathBuf,
235 pub provenance_path: PathBuf,
237 pub product: Vec<u8>,
239 pub archive: Vec<u8>,
241 pub provenance: Vec<u8>,
243 }
244
245 #[derive(Debug, Clone)]
247 pub struct ExactProductCache {
248 stable_path: PathBuf,
249 identity: ProductIdentity,
250 source: DistributionSource,
251 }
252
253 pub struct ExactCacheGuard {
255 lock_file: File,
256 stable_path: PathBuf,
257 }
258
259 impl Drop for ExactCacheGuard {
260 fn drop(&mut self) {
261 let _ = FileExt::unlock(&self.lock_file);
262 }
263 }
264
265 impl ExactProductCache {
266 pub fn new(
268 stable_path: impl Into<PathBuf>,
269 identity: ProductIdentity,
270 source: DistributionSource,
271 ) -> Result<Self, ExactCacheError> {
272 identity.validate()?;
273 let stable_path = stable_path.into();
274 if stable_path.file_name().is_none() || stable_path.parent().is_none() {
275 return Err(ExactCacheError::InvalidCommit("stable product path"));
276 }
277 Ok(Self {
278 stable_path,
279 identity,
280 source,
281 })
282 }
283
284 #[must_use]
286 pub fn stable_path(&self) -> &Path {
287 &self.stable_path
288 }
289
290 pub fn lock(&self, timeout: Duration) -> Result<ExactCacheGuard, ExactCacheError> {
292 ensure_supported_platform()?;
293 let parent = self
294 .stable_path
295 .parent()
296 .ok_or(ExactCacheError::InvalidCommit("stable product parent"))?;
297 durable_create_dir_all(parent)?;
298 let lock_path = parent.join(LOCK_FILENAME);
299 let lock_file = OpenOptions::new()
300 .create(true)
301 .truncate(false)
302 .read(true)
303 .write(true)
304 .open(lock_path)
305 .map_err(|source| io("open lock", source))?;
306 lock_file
307 .sync_all()
308 .map_err(|source| io("sync lock", source))?;
309 sync_directory(parent)?;
310 let deadline = Instant::now()
311 .checked_add(timeout)
312 .ok_or(ExactCacheError::LockTimeout)?;
313 loop {
314 match lock_file.try_lock_exclusive() {
315 Ok(()) => {
316 return Ok(ExactCacheGuard {
317 lock_file,
318 stable_path: self.stable_path.clone(),
319 });
320 }
321 Err(error) if error.kind() == ErrorKind::WouldBlock => {
322 let now = Instant::now();
323 if now >= deadline {
324 return Err(ExactCacheError::LockTimeout);
325 }
326 thread::sleep((deadline - now).min(Duration::from_millis(10)));
327 }
328 Err(source) => return Err(io("lock", source)),
329 }
330 }
331 }
332
333 pub fn read(&self) -> Result<Option<CommittedExactCacheEntry>, ExactCacheError> {
338 let marker_path = self.marker_path();
339 for _ in 0..16 {
340 let marker = match fs::read(&marker_path) {
341 Ok(bytes) => bytes,
342 Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
343 Err(source) => return Err(io("read marker", source)),
344 };
345 test_read_barrier();
346 match self.read_committed_entry(&marker) {
347 Ok(entry) => return Ok(Some(entry)),
348 Err(error) => match fs::read(&marker_path) {
349 Ok(current) if current != marker => continue,
350 Err(current_error) if current_error.kind() == ErrorKind::NotFound => {
351 continue;
352 }
353 _ => return Err(error),
354 },
355 }
356 }
357 Err(ExactCacheError::InvalidCommit(
358 "commit changed repeatedly during read",
359 ))
360 }
361
362 fn read_committed_entry(
363 &self,
364 marker: &[u8],
365 ) -> Result<CommittedExactCacheEntry, ExactCacheError> {
366 let record: CommitRecord = serde_json::from_slice(marker)
367 .map_err(|_| ExactCacheError::InvalidCommit("malformed JSON"))?;
368 validate_entry_id(&record.entry)?;
369 let paths = self.entry_paths(&record.entry)?;
370 let product = fs::read(&paths.product).map_err(|source| io("read product", source))?;
371 let archive = fs::read(&paths.archive).map_err(|source| io("read archive", source))?;
372 let provenance =
373 fs::read(&paths.provenance).map_err(|source| io("read provenance", source))?;
374 let verified = verify_commit_record(
375 &self.identity,
376 self.source,
377 marker,
378 &product,
379 &archive,
380 &provenance,
381 )?;
382 Ok(CommittedExactCacheEntry {
383 entry_id: verified.entry_id,
384 product_path: paths.product,
385 archive_path: paths.archive,
386 provenance_path: paths.provenance,
387 product,
388 archive,
389 provenance,
390 })
391 }
392
393 pub fn publish(
395 &self,
396 guard: &ExactCacheGuard,
397 product: &[u8],
398 archive: &[u8],
399 provenance: &[u8],
400 ) -> Result<CommittedExactCacheEntry, ExactCacheError> {
401 self.require_guard(guard)?;
402 ensure_supported_platform()?;
403 let control = self.control_directory();
404 let entries = control.join("entries");
405 durable_create_dir_all(&entries)?;
406 let entry_id = self.allocate_entry_id(&entries)?;
407 let paths = self.entry_paths(&entry_id)?;
408 let entry_directory = paths
409 .product
410 .parent()
411 .ok_or(ExactCacheError::InvalidCommit("entry parent"))?;
412 sync_directory(&entries)?;
413 let marker_temp =
414 control.join(format!(".{EXACT_CACHE_MARKER_FILENAME}.{entry_id}.tmp"));
415 let marker = build_commit_record(
416 &self.identity,
417 self.source,
418 &entry_id,
419 product,
420 archive,
421 provenance,
422 )?;
423 let result: Result<(), ExactCacheError> = (|| {
424 write_exclusive(&paths.product, product)?;
425 test_failpoint("after_payload");
426 write_exclusive(&paths.archive, archive)?;
427 test_failpoint("after_archive");
428 write_exclusive(&paths.provenance, provenance)?;
429 test_failpoint("after_metadata");
430 sync_directory(entry_directory)?;
431 sync_directory(&entries)?;
432 test_failpoint("after_entry_sync");
433 write_exclusive(&marker_temp, &marker)?;
434 test_failpoint("after_marker_write");
435 fs::rename(&marker_temp, self.marker_path())
436 .map_err(|source| io("rename marker", source))?;
437 test_failpoint("after_marker_rename");
438 sync_directory(&control)?;
439 test_failpoint("after_commit_sync");
440 Ok(())
441 })();
442 if result.is_err() {
443 let _ = fs::remove_file(&marker_temp);
444 if self.current_entry_id().as_deref() != Some(entry_id.as_str()) {
445 let _ = fs::remove_dir_all(entry_directory);
446 }
447 }
448 result?;
449 Ok(CommittedExactCacheEntry {
450 entry_id,
451 product_path: paths.product,
452 archive_path: paths.archive,
453 provenance_path: paths.provenance,
454 product: product.to_vec(),
455 archive: archive.to_vec(),
456 provenance: provenance.to_vec(),
457 })
458 }
459
460 pub fn cleanup_abandoned(&self, guard: &ExactCacheGuard) -> Result<(), ExactCacheError> {
462 self.require_guard(guard)?;
463 let control = self.control_directory();
464 let entries = control.join("entries");
465 let current = match fs::read(self.marker_path()) {
466 Ok(marker) => {
467 let record: CommitRecord = match serde_json::from_slice(&marker) {
468 Ok(record) => record,
469 Err(_) => return Ok(()),
470 };
471 if validate_entry_id(&record.entry).is_err() {
472 return Ok(());
473 }
474 Some(record.entry)
475 }
476 Err(error) if error.kind() == ErrorKind::NotFound => None,
477 Err(_) => return Ok(()),
478 };
479 if let Ok(children) = fs::read_dir(&entries) {
480 for child in children.flatten() {
481 let name = child.file_name();
482 if current.as_deref() != name.to_str() {
483 let _ = fs::remove_dir_all(child.path());
484 }
485 }
486 }
487 if let Ok(children) = fs::read_dir(&control) {
488 let prefix = format!(".{EXACT_CACHE_MARKER_FILENAME}.");
489 for child in children.flatten() {
490 let name = child.file_name();
491 let Some(name) = name.to_str() else {
492 continue;
493 };
494 if name.starts_with(&prefix) && name.ends_with(".tmp") {
495 let _ = fs::remove_file(child.path());
496 }
497 }
498 }
499 Ok(())
500 }
501
502 fn require_guard(&self, guard: &ExactCacheGuard) -> Result<(), ExactCacheError> {
503 if guard.stable_path == self.stable_path {
504 Ok(())
505 } else {
506 Err(ExactCacheError::InvalidCommit("cache lock scope"))
507 }
508 }
509
510 fn control_directory(&self) -> PathBuf {
511 self.stable_path
512 .parent()
513 .expect("validated cache path has parent")
514 .join(EXACT_CACHE_CONTROL_DIRECTORY)
515 }
516
517 fn marker_path(&self) -> PathBuf {
518 self.control_directory().join(EXACT_CACHE_MARKER_FILENAME)
519 }
520
521 fn entry_paths(&self, entry_id: &str) -> Result<EntryPaths, ExactCacheError> {
522 validate_entry_id(entry_id)?;
523 let filename = self
524 .stable_path
525 .file_name()
526 .ok_or(ExactCacheError::InvalidCommit("stable product filename"))?;
527 let entry = self.control_directory().join("entries").join(entry_id);
528 let product = entry.join(filename);
529 let archive = entry.join(format!("{}.archive", filename.to_string_lossy()));
530 let provenance = entry.join(format!("{}.provenance.json", filename.to_string_lossy()));
531 Ok(EntryPaths {
532 product,
533 archive,
534 provenance,
535 })
536 }
537
538 fn allocate_entry_id(&self, entries: &Path) -> Result<String, ExactCacheError> {
539 for _ in 0..128 {
540 let mut random = [0_u8; 16];
541 getrandom::getrandom(&mut random).map_err(|error| {
542 io("random entry id", std::io::Error::other(error.to_string()))
543 })?;
544 let mut entry_id = String::with_capacity(32);
545 for byte in random {
546 write!(&mut entry_id, "{byte:02x}").expect("writing to String cannot fail");
547 }
548 match fs::create_dir(entries.join(&entry_id)) {
549 Ok(()) => return Ok(entry_id),
550 Err(error) if error.kind() == ErrorKind::AlreadyExists => continue,
551 Err(source) => return Err(io("create entry", source)),
552 }
553 }
554 Err(ExactCacheError::InvalidCommit("entry identifier collision"))
555 }
556
557 fn current_entry_id(&self) -> Option<String> {
558 let marker = fs::read(self.marker_path()).ok()?;
559 let record: CommitRecord = serde_json::from_slice(&marker).ok()?;
560 validate_entry_id(&record.entry).ok()?;
561 Some(record.entry)
562 }
563 }
564
565 struct EntryPaths {
566 product: PathBuf,
567 archive: PathBuf,
568 provenance: PathBuf,
569 }
570
571 fn io(operation: &'static str, source: std::io::Error) -> ExactCacheError {
572 ExactCacheError::Io { operation, source }
573 }
574
575 fn ensure_supported_platform() -> Result<(), ExactCacheError> {
576 if cfg!(any(target_os = "linux", target_os = "macos")) {
577 Ok(())
578 } else {
579 Err(ExactCacheError::UnsupportedPlatform)
580 }
581 }
582
583 fn durable_create_dir_all(path: &Path) -> Result<(), ExactCacheError> {
584 if path.is_dir() {
585 return Ok(());
586 }
587 let mut missing = Vec::new();
588 let mut cursor = path;
589 while !cursor.exists() {
590 missing.push(cursor.to_path_buf());
591 cursor = cursor
592 .parent()
593 .ok_or(ExactCacheError::InvalidCommit("cache directory parent"))?;
594 }
595 for directory in missing.iter().rev() {
596 match fs::create_dir(directory) {
597 Ok(()) => {}
598 Err(error) if error.kind() == ErrorKind::AlreadyExists && directory.is_dir() => {}
599 Err(source) => return Err(io("create directory", source)),
600 }
601 let parent = directory
602 .parent()
603 .ok_or(ExactCacheError::InvalidCommit("cache directory parent"))?;
604 sync_directory(parent)?;
605 }
606 Ok(())
607 }
608
609 fn write_exclusive(path: &Path, bytes: &[u8]) -> Result<(), ExactCacheError> {
610 let mut file = OpenOptions::new()
611 .create_new(true)
612 .write(true)
613 .open(path)
614 .map_err(|source| io("create immutable file", source))?;
615 file.write_all(bytes)
616 .map_err(|source| io("write immutable file", source))?;
617 file.sync_all()
618 .map_err(|source| io("sync immutable file", source))
619 }
620
621 fn sync_directory(path: &Path) -> Result<(), ExactCacheError> {
622 File::open(path)
623 .and_then(|directory| directory.sync_all())
624 .map_err(|source| io("sync directory", source))
625 }
626
627 #[cfg(feature = "exact-cache-test-failpoints")]
628 fn test_failpoint(name: &str) {
629 if std::env::var_os("SIDEREON_TEST_EXACT_CACHE_FAILPOINT").as_deref()
630 == Some(std::ffi::OsStr::new(name))
631 {
632 std::process::exit(86);
633 }
634 }
635
636 #[cfg(not(feature = "exact-cache-test-failpoints"))]
637 fn test_failpoint(_name: &str) {}
638
639 #[cfg(feature = "exact-cache-test-failpoints")]
640 fn test_read_barrier() {
641 static ONCE: std::sync::Once = std::sync::Once::new();
642 ONCE.call_once(|| {
643 let Some(barrier) = std::env::var_os("SIDEREON_TEST_EXACT_CACHE_READ_BARRIER") else {
644 return;
645 };
646 let barrier = PathBuf::from(barrier);
647 fs::write(barrier.with_extension("ready"), b"ready")
648 .expect("write exact-cache read barrier");
649 let release = barrier.with_extension("release");
650 let deadline = Instant::now() + Duration::from_secs(10);
651 while !release.exists() {
652 assert!(
653 Instant::now() < deadline,
654 "timed out waiting for exact-cache read barrier"
655 );
656 thread::sleep(Duration::from_millis(5));
657 }
658 });
659 }
660
661 #[cfg(not(feature = "exact-cache-test-failpoints"))]
662 fn test_read_barrier() {}
663}
664
665#[cfg(not(target_arch = "wasm32"))]
666pub use native::{CommittedExactCacheEntry, ExactCacheGuard, ExactProductCache};
667
668#[cfg(test)]
669mod tests {
670 use super::*;
671 use crate::data::{product, AnalysisCenter, ProductDate, ProductType};
672
673 fn identity() -> ProductIdentity {
674 product(
675 AnalysisCenter::CodUlt,
676 ProductType::Sp3,
677 ProductDate::new(2026, 7, 16).expect("date"),
678 Some("05M"),
679 Some("0000"),
680 )
681 .expect("product")
682 .identity()
683 .expect("identity")
684 }
685
686 #[test]
687 fn commit_binds_every_byte_group_identity_and_source() {
688 let identity = identity();
689 let marker = build_commit_record(
690 &identity,
691 DistributionSource::Direct,
692 "0123456789abcdef0123456789abcdef",
693 b"product",
694 b"archive",
695 b"provenance",
696 )
697 .expect("commit");
698 let verified = verify_commit_record(
699 &identity,
700 DistributionSource::Direct,
701 &marker,
702 b"product",
703 b"archive",
704 b"provenance",
705 )
706 .expect("verify");
707 assert_eq!(verified.entry_id, "0123456789abcdef0123456789abcdef");
708
709 for (product, archive, provenance) in [
710 (&b"changed"[..], &b"archive"[..], &b"provenance"[..]),
711 (&b"product"[..], &b"changed"[..], &b"provenance"[..]),
712 (&b"product"[..], &b"archive"[..], &b"changed"[..]),
713 ] {
714 assert!(verify_commit_record(
715 &identity,
716 DistributionSource::Direct,
717 &marker,
718 product,
719 archive,
720 provenance,
721 )
722 .is_err());
723 }
724 assert!(verify_commit_record(
725 &identity,
726 DistributionSource::NasaCddis,
727 &marker,
728 b"product",
729 b"archive",
730 b"provenance",
731 )
732 .is_err());
733 }
734
735 #[test]
736 fn malformed_entry_ids_are_rejected() {
737 for entry in [
738 "",
739 "ABCDEF0123456789ABCDEF0123456789",
740 "0123456789abcdef0123456789abcdeg",
741 "0123456789abcdef",
742 ] {
743 assert!(build_commit_record(
744 &identity(),
745 DistributionSource::Direct,
746 entry,
747 b"product",
748 b"archive",
749 b"provenance",
750 )
751 .is_err());
752 }
753 }
754}