concinnity_host/store/blob/
mod.rs

1//! Runtime blob access: the state root's `data/` path layout, the payload
2//! residency store, and all blob file I/O. The concinnity-blob crate owns the
3//! format contract (schema, header, version, bytes <-> metadata) and is
4//! deliberately I/O-free, so every read below is `fs` here plus a pure parse
5//! there. Blob data is read-only at runtime; concinnity-cook writes what
6//! `concinnity_core::blob::encode_cnb` returns.
7use std::fs;
8use std::io::Read;
9use std::path::{Path, PathBuf};
10use std::sync::{Mutex, OnceLock};
11
12pub use concinnity_core::blob::{BLOB_MAGIC, HEADER_SIZE, WorldManifest};
13use concinnity_core::blob::{BlobError, parse_cnb, parse_payload_section_start, payload_section};
14use concinnity_core::result::CnResult;
15
16mod data;
17
18pub use concinnity_core::SCHEMA_VERSION;
19pub use concinnity_core::ecs::{BlobAssetDef, BlobMeta, ResourceRecord};
20pub use data::BlobData;
21
22// The primary blob this process reads, named by whatever anchored it. Process
23// state because payloads stream off disk long after startup: a locator resolved
24// mid-frame has no caller to carry the layout down from.
25fn anchored_primary() -> &'static Mutex<Option<PathBuf>> {
26    static PRIMARY: OnceLock<Mutex<Option<PathBuf>>> = OnceLock::new();
27    PRIMARY.get_or_init(|| Mutex::new(None))
28}
29
30// Anchor the process's blob layout on `primary`: that file is blob 0 and its
31// siblings named by index are the overflow payload blobs. Reached only through
32// `load_raw_at`, so a process addresses the blobs it actually opened.
33fn anchor(primary: &Path) {
34    *anchored_primary().lock().unwrap() = Some(primary.to_path_buf());
35}
36
37// Blob 0 is the primary blob; every other index names an overflow sibling.
38const PRIMARY_INDEX: u32 = 0;
39
40/// The primary blob inside `data_dir`: blob 0, holding the metadata block plus
41/// the first payload section. What a host opens when it reads a state tree's
42/// `data/`, and what a build writes there.
43pub fn primary_in(data_dir: &Path) -> PathBuf {
44    data_dir.join(PRIMARY_INDEX.to_string())
45}
46
47/// Format a blob file path for a given index. Blob 0 is the primary blob
48/// [`load_raw_at`] opened (the metadata block plus the first payload section);
49/// higher indices are overflow payload blobs, which are always its siblings.
50/// The format crate is path-agnostic; this layout knowledge stays here.
51///
52/// `None` before any load, so there is no layout to resolve against.
53pub fn blob_path(index: u32) -> Option<String> {
54    let primary = anchored_primary().lock().unwrap().clone();
55    resolve_blob_path(primary.as_deref(), index)
56}
57
58// Pure resolution split out so the sibling naming is unit-testable without the
59// process-global anchor.
60fn resolve_blob_path(primary: Option<&Path>, index: u32) -> Option<String> {
61    let primary = primary?;
62    let path = if index == PRIMARY_INDEX {
63        primary.to_path_buf()
64    } else {
65        primary
66            .parent()
67            .map_or_else(|| PathBuf::from("."), Path::to_path_buf)
68            .join(index.to_string())
69    };
70    Some(path.to_string_lossy().into_owned())
71}
72
73/// Read and deserialize a blob's metadata section (component defs + resource
74/// records). Returns (meta, payload_start_offset).
75pub fn read_cnb(path: &str) -> Result<(BlobMeta, usize), CnResult> {
76    let data = read_file(path)?;
77    parse_cnb::<BlobMeta>(SCHEMA_VERSION, &data).map_err(|e| report(path, e))
78}
79
80/// Byte offset within a blob file at which its payload section begins. Reads
81/// only the header; the disk-backed streaming source uses it to turn a
82/// `PayloadLocator` offset into an absolute file offset.
83/// Used only by the Metal-driven disk-backed streaming source for now
84/// (Vulkan/DirectX streaming catch-up is a follow-up).
85pub fn payload_section_start(path: &str) -> Result<u64, CnResult> {
86    let mut file = fs::File::open(path).map_err(|e| {
87        tracing::error!("Failed to open {}: {}", path, e);
88        CnResult::FileIo
89    })?;
90    let mut header = [0u8; HEADER_SIZE];
91    file.read_exact(&mut header).map_err(|e| {
92        tracing::error!("Failed to read header of {}: {}", path, e);
93        CnResult::FileIo
94    })?;
95    parse_payload_section_start::<BlobMeta>(&header).map_err(|e| report(path, e))
96}
97
98// Read just the payload section of a blob file into memory.
99fn read_payload_section(path: &str) -> Result<Vec<u8>, CnResult> {
100    let data = read_file(path)?;
101    Ok(payload_section(&data).to_vec())
102}
103
104fn read_file(path: &str) -> Result<Vec<u8>, CnResult> {
105    fs::read(path).map_err(|e| {
106        tracing::error!("Failed to read {}: {}", path, e);
107        CnResult::FileIo
108    })
109}
110
111// Log a format failure against the file it came from. The format crate has no
112// path to name, so the diagnostic belongs here.
113fn report(path: &str, e: BlobError) -> CnResult {
114    match e {
115        BlobError::TooShort => tracing::error!("{}: file too short", path),
116        BlobError::BadMagic => tracing::error!("{}: bad magic", path),
117        BlobError::ValidityMismatch(_) => tracing::error!(
118            "{}: world data was built by a different version of the engine",
119            path
120        ),
121        BlobError::TruncatedMeta => tracing::error!("{}: truncated metadata section", path),
122        BlobError::Decode => tracing::error!("{}: failed to deserialize metadata", path),
123        BlobError::TrailingMeta(n) => tracing::error!(
124            "{}: metadata left {} unread bytes; world data was built by a different version of the engine",
125            path,
126            n
127        ),
128        BlobError::Encode => tracing::error!("{}: failed to serialize metadata", path),
129    }
130    CnResult::FileIo
131}
132
133/// Load the blob file at `primary` and the payload store around it, anchoring
134/// the process's blob layout on it, so a world written to `data/0` reads
135/// `data/1`, `data/2`, ... beside it. The anchor outlives the call because
136/// payloads stream off disk long after startup: a locator resolved mid-frame
137/// has no caller to carry the layout down from.
138///
139/// Only blob 0's payload section is read here; overflow blobs (named by the
140/// manifest's `max_blob_index`) start unloaded and `BlobData::read()` pulls
141/// each from disk the first time a locator needs it. Defs are not resolved into
142/// runtime `Asset`s: that resolution depends on the client runtime registry, so
143/// it lives in the client `blob::load` shim.
144pub fn load_raw_at(primary: &Path) -> Result<(BlobMeta, BlobData), CnResult> {
145    anchor(primary);
146    load_raw_from(blob_path)
147}
148
149// `load_raw` against an injected layout, so the eager/deferred split can be
150// exercised without the process-global data-dir anchor.
151fn load_raw_from(
152    blob_path: impl Fn(u32) -> Option<String>,
153) -> Result<(BlobMeta, BlobData), CnResult> {
154    let (meta, _payload_start) = read_cnb(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
155
156    // Cook derives the manifest from the very streams it summarizes, so a
157    // mismatch means a corrupt or hand-edited blob.
158    debug_assert_eq!(
159        meta.manifest,
160        WorldManifest::from_records(&meta.defs, &meta.resources),
161        "blob manifest does not match its record streams"
162    );
163
164    let blob0_payload = read_payload_section(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
165    tracing::debug!("Loaded blob 0 payload ({} bytes)", blob0_payload.len());
166    let overflow_paths = (1..=meta.manifest.max_blob_index)
167        .map(|i| blob_path(i).ok_or(CnResult::NoStateRoot))
168        .collect::<Result<Vec<_>, _>>()?;
169
170    let blob_data = BlobData::from_blob_files(blob0_payload, overflow_paths);
171    Ok((meta, blob_data))
172}
173
174#[cfg(test)]
175mod tests {
176    use super::*;
177    use concinnity_core::blob::encode_cnb;
178
179    #[test]
180    fn an_anchored_primary_owns_blob_zero_and_its_siblings() {
181        // Blob 0 is the file named verbatim (whatever it is called); every
182        // overflow blob is its sibling named by index. Built through `join` so
183        // the separator is the platform's.
184        let primary = Path::new("out").join("blobs").join("0");
185        assert_eq!(
186            resolve_blob_path(Some(&primary), 0).as_deref(),
187            Some(&*primary.to_string_lossy())
188        );
189        assert_eq!(
190            resolve_blob_path(Some(&primary), 2),
191            Some(
192                Path::new("out")
193                    .join("blobs")
194                    .join("2")
195                    .to_string_lossy()
196                    .into_owned()
197            )
198        );
199
200        // A bare file name hangs its siblings off the working directory.
201        assert_eq!(
202            resolve_blob_path(Some(Path::new("0")), 1).as_deref(),
203            Some("1")
204        );
205
206        // A state tree's `data/` is just a primary named `<tree>/data/0`, so
207        // the overflow blobs land beside it the same way.
208        let tree = crate::store::paths::StateTree::at(Path::new("/proj"));
209        let data = tree.data_dir().join("0");
210        assert_eq!(
211            resolve_blob_path(Some(&data), 3),
212            Some(tree.data_dir().join("3").to_string_lossy().into_owned())
213        );
214    }
215
216    // With nothing anchored there is no layout to resolve against, which is
217    // what turns a blob read into `NoStateRoot` rather than a read of some path
218    // relative to the working directory.
219    #[test]
220    fn without_an_anchor_there_is_no_path() {
221        assert_eq!(resolve_blob_path(None, 0), None);
222        assert_eq!(resolve_blob_path(None, 3), None);
223    }
224
225    #[test]
226    fn format_failures_fold_onto_file_io() {
227        assert_eq!(report("x.cnb", BlobError::BadMagic), CnResult::FileIo);
228        assert_eq!(
229            report("x.cnb", BlobError::ValidityMismatch(99)),
230            CnResult::FileIo
231        );
232    }
233
234    #[test]
235    fn read_cnb_errors_on_a_missing_file() {
236        assert_eq!(
237            read_cnb("/nonexistent/cn/blob/path.cnb"),
238            Err(CnResult::FileIo)
239        );
240    }
241
242    #[test]
243    fn read_payload_section_returns_empty_for_a_short_file() {
244        let dir = tempfile::tempdir().unwrap();
245        let path = dir.path().join("short").to_string_lossy().into_owned();
246        std::fs::write(&path, vec![0u8; HEADER_SIZE - 1]).unwrap();
247        assert!(read_payload_section(&path).unwrap().is_empty());
248    }
249
250    #[test]
251    fn payload_section_start_skips_header_and_meta() {
252        let dir = tempfile::tempdir().unwrap();
253        let path = dir.path().join("0").to_string_lossy().into_owned();
254        let image = encode_cnb(SCHEMA_VERSION, &BlobMeta::default(), b"payloadbytes").unwrap();
255        std::fs::write(&path, &image).unwrap();
256
257        let start = payload_section_start(&path).expect("section start");
258        assert_eq!(&image[start as usize..], b"payloadbytes");
259    }
260
261    #[test]
262    fn payload_section_start_rejects_bad_magic() {
263        let dir = tempfile::tempdir().unwrap();
264        let path = dir.path().join("bad").to_string_lossy().into_owned();
265        std::fs::write(&path, vec![0u8; HEADER_SIZE]).unwrap();
266        assert_eq!(payload_section_start(&path), Err(CnResult::FileIo));
267    }
268
269    #[test]
270    fn load_raw_reads_blob0_eagerly_and_defers_overflow() {
271        use concinnity_core::ecs::{AssetKind, PayloadLocator};
272
273        let dir = tempfile::tempdir().unwrap();
274        let path_for = |idx: u32| {
275            Some(
276                dir.path()
277                    .join(idx.to_string())
278                    .to_string_lossy()
279                    .into_owned(),
280            )
281        };
282
283        // Blob 0: one def whose payload lives in overflow blob 1. The manifest
284        // is derived exactly as cook derives it; `load_raw` trusts its
285        // `max_blob_index` to name the overflow file.
286        let defs = vec![BlobAssetDef {
287            name: None,
288            kind: AssetKind::Component,
289            discriminant: 1,
290            args_bytes: Vec::new(),
291            payload: Some(PayloadLocator {
292                blob_index: 1,
293                offset: 0,
294                len: 8,
295            }),
296        }];
297        let meta = BlobMeta {
298            manifest: WorldManifest::from_records(&defs, &[]),
299            defs,
300            resources: Vec::new(),
301            scene_groups: Vec::new(),
302            mesh_bounds: Vec::new(),
303            physics_budget: None,
304        };
305        std::fs::write(
306            path_for(0).unwrap(),
307            encode_cnb(SCHEMA_VERSION, &meta, b"primary").unwrap(),
308        )
309        .unwrap();
310        std::fs::write(
311            path_for(1).unwrap(),
312            encode_cnb(SCHEMA_VERSION, &BlobMeta::default(), b"overflow").unwrap(),
313        )
314        .unwrap();
315
316        let (meta, mut bd) = load_raw_from(path_for).expect("load");
317        assert_eq!(meta.defs.len(), 1);
318        assert!(meta.resources.is_empty());
319        assert_eq!(meta.manifest.component_counts, vec![(1, 1)]);
320        assert!(bd.disk_backed());
321        // Blob 0 resident, blob 1 deferred until its first read.
322        assert!(bd.is_loaded(0));
323        assert!(!bd.is_loaded(1));
324        let loc = meta.defs[0].payload.clone().unwrap();
325        assert_eq!(bd.read(&loc).expect("overflow read"), b"overflow");
326        assert!(bd.is_loaded(1));
327    }
328
329    // A layout that resolves to nothing is the uninstalled-state-root case, and
330    // it has to name itself rather than folding onto a file-not-found.
331    #[test]
332    fn load_raw_without_a_layout_reports_no_state_root() {
333        assert_eq!(load_raw_from(|_| None).err(), Some(CnResult::NoStateRoot));
334    }
335}
concinnity_host/store/blob/mod.rs

concinnity_host/store/blob/
mod.rs
