concinnity_host/store/blob/
mod.rs

1//! Runtime blob access: the state root's `data/` path layout, the payload
2//! residency store, and all blob file I/O. The concinnity-blob crate owns the
3//! format contract (schema, header, version, bytes <-> metadata) and is
4//! deliberately I/O-free, so every read below is `fs` here plus a pure parse
5//! there. Blob data is read-only at runtime; concinnity-cook writes what
6//! `concinnity_core::blob::encode_cnb` returns.
7use std::fs;
8use std::io::Read;
9use std::path::{Path, PathBuf};
10use std::sync::{Mutex, OnceLock};
11
12pub use concinnity_core::blob::{BLOB_MAGIC, HEADER_SIZE, WorldManifest};
13use concinnity_core::blob::{BlobError, parse_cnb, parse_payload_section_start, payload_section};
14use concinnity_core::result::CnResult;
15
16mod data;
17
18pub use concinnity_core::SCHEMA_VERSION;
19pub use concinnity_core::ecs::{BlobAssetDef, BlobMeta, ResourceRecord};
20pub use data::BlobData;
21
22// The primary blob this process reads, named by whatever anchored it. Process
23// state because payloads stream off disk long after startup: a locator resolved
24// mid-frame has no caller to carry the layout down from.
25fn anchored_primary() -> &'static Mutex<Option<PathBuf>> {
26    static PRIMARY: OnceLock<Mutex<Option<PathBuf>>> = OnceLock::new();
27    PRIMARY.get_or_init(|| Mutex::new(None))
28}
29
30// Anchor the process's blob layout on `primary`: that file is blob 0 and its
31// siblings named by index are the overflow payload blobs. Reached only through
32// `load_raw_at`, so a process addresses the blobs it actually opened.
33fn anchor(primary: &Path) {
34    *anchored_primary().lock().unwrap() = Some(primary.to_path_buf());
35}
36
37// Blob 0 is the primary blob; every other index names an overflow sibling.
38const PRIMARY_INDEX: u32 = 0;
39
40/// The primary blob inside `data_dir`: blob 0, holding the metadata block plus
41/// the first payload section. What a host opens when it reads a state tree's
42/// `data/`, and what a build writes there.
43pub fn primary_in(data_dir: &Path) -> PathBuf {
44    data_dir.join(PRIMARY_INDEX.to_string())
45}
46
47/// Format a blob file path for a given index. Blob 0 is the primary blob
48/// [`load_raw_at`] opened (the metadata block plus the first payload section);
49/// higher indices are overflow payload blobs, which are always its siblings.
50/// The format crate is path-agnostic; this layout knowledge stays here.
51///
52/// `None` before any load, so there is no layout to resolve against.
53pub fn blob_path(index: u32) -> Option<String> {
54    let primary = anchored_primary().lock().unwrap().clone();
55    resolve_blob_path(primary.as_deref(), index)
56}
57
58// Pure resolution split out so the sibling naming is unit-testable without the
59// process-global anchor.
60fn resolve_blob_path(primary: Option<&Path>, index: u32) -> Option<String> {
61    let primary = primary?;
62    let path = if index == PRIMARY_INDEX {
63        primary.to_path_buf()
64    } else {
65        primary
66            .parent()
67            .map_or_else(|| PathBuf::from("."), Path::to_path_buf)
68            .join(index.to_string())
69    };
70    Some(path.to_string_lossy().into_owned())
71}
72
73/// Read and deserialize a blob's metadata section (component defs + resource
74/// records). Returns (meta, payload_start_offset).
75pub fn read_cnb(path: &str) -> Result<(BlobMeta, usize), CnResult> {
76    let data = read_file(path)?;
77    parse_cnb::<BlobMeta>(SCHEMA_VERSION, &data).map_err(|e| report(path, e))
78}
79
80/// Byte offset within a blob file at which its payload section begins. Reads
81/// only the header; the disk-backed streaming source uses it to turn a
82/// `PayloadLocator` offset into an absolute file offset.
83/// Used only by the Metal-driven disk-backed streaming source for now
84/// (Vulkan/DirectX streaming catch-up is a follow-up).
85pub fn payload_section_start(path: &str) -> Result<u64, CnResult> {
86    let mut file = fs::File::open(path).map_err(|e| {
87        tracing::error!("Failed to open {}: {}", path, e);
88        CnResult::FileIo
89    })?;
90    let mut header = [0u8; HEADER_SIZE];
91    file.read_exact(&mut header).map_err(|e| {
92        tracing::error!("Failed to read header of {}: {}", path, e);
93        CnResult::FileIo
94    })?;
95    parse_payload_section_start::<BlobMeta>(&header).map_err(|e| report(path, e))
96}
97
98// Read just the payload section of a blob file into memory.
99fn read_payload_section(path: &str) -> Result<Vec<u8>, CnResult> {
100    let data = read_file(path)?;
101    Ok(payload_section(&data).to_vec())
102}
103
104fn read_file(path: &str) -> Result<Vec<u8>, CnResult> {
105    fs::read(path).map_err(|e| {
106        tracing::error!("Failed to read {}: {}", path, e);
107        CnResult::FileIo
108    })
109}
110
111// Log a format failure against the file it came from. The format crate has no
112// path to name, so the diagnostic belongs here.
113fn report(path: &str, e: BlobError) -> CnResult {
114    match e {
115        BlobError::TooShort => tracing::error!("{}: file too short", path),
116        BlobError::BadMagic => tracing::error!("{}: bad magic", path),
117        BlobError::ValidityMismatch(_) => tracing::error!(
118            "{}: world data was built by a different version of the engine",
119            path
120        ),
121        BlobError::TruncatedMeta => tracing::error!("{}: truncated metadata section", path),
122        BlobError::Decode => tracing::error!("{}: failed to deserialize metadata", path),
123        BlobError::TrailingMeta(n) => tracing::error!(
124            "{}: metadata left {} unread bytes; world data was built by a different version of the engine",
125            path,
126            n
127        ),
128        BlobError::Encode => tracing::error!("{}: failed to serialize metadata", path),
129    }
130    CnResult::FileIo
131}
132
133/// Load the blob file at `primary` and the payload store around it, anchoring
134/// the process's blob layout on it, so a world written to `data/0` reads
135/// `data/1`, `data/2`, ... beside it. The anchor outlives the call because
136/// payloads stream off disk long after startup: a locator resolved mid-frame
137/// has no caller to carry the layout down from.
138///
139/// Only blob 0's payload section is read here; overflow blobs (named by the
140/// manifest's `max_blob_index`) start unloaded and `BlobData::read()` pulls
141/// each from disk the first time a locator needs it. Defs are not resolved into
142/// runtime `Asset`s: that resolution depends on the client runtime registry, so
143/// it lives in the client `blob::load` shim.
144pub fn load_raw_at(primary: &Path) -> Result<(BlobMeta, BlobData), CnResult> {
145    anchor(primary);
146    load_raw_from(blob_path)
147}
148
149// `load_raw` against an injected layout, so the eager/deferred split can be
150// exercised without the process-global data-dir anchor.
151fn load_raw_from(
152    blob_path: impl Fn(u32) -> Option<String>,
153) -> Result<(BlobMeta, BlobData), CnResult> {
154    let (meta, _payload_start) = read_cnb(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
155
156    // Cook derives the manifest from the very streams it summarizes, so a
157    // mismatch means a corrupt or hand-edited blob.
158    debug_assert_eq!(
159        meta.manifest,
160        WorldManifest::from_records(&meta.defs, &meta.resources),
161        "blob manifest does not match its record streams"
162    );
163
164    let blob0_payload = read_payload_section(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
165    tracing::debug!("Loaded blob 0 payload ({} bytes)", blob0_payload.len());
166    let overflow_paths = (1..=meta.manifest.max_blob_index)
167        .map(|i| blob_path(i).ok_or(CnResult::NoStateRoot))
168        .collect::<Result<Vec<_>, _>>()?;
169
170    let blob_data = BlobData::from_blob_files(blob0_payload, overflow_paths);
171    Ok((meta, blob_data))
172}
173
174/// Number of Texture resource records in the primary blob's metadata, read
175/// without loading any payload. This is the compiled world's texture-table
176/// length; `cn export` uses it to precompile the built-in shaders whose bindless
177/// texture pool is sized per world.
178pub fn texture_resource_count() -> Result<usize, CnResult> {
179    let (meta, _) = read_cnb(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
180    let tag = concinnity_core::ecs::ResourceKind::Texture as u8;
181    Ok(meta
182        .resources
183        .iter()
184        .filter(|r| r.resource_kind == tag)
185        .count())
186}
187
188/// Load defs without resolving (for callers that apply overlays first)
189pub fn load_defs() -> Result<Vec<BlobAssetDef>, CnResult> {
190    read_cnb(&blob_path(0).ok_or(CnResult::NoStateRoot)?).map(|(meta, _)| meta.defs)
191}
192
193#[cfg(test)]
194mod tests {
195    use super::*;
196    use concinnity_core::blob::encode_cnb;
197
198    #[test]
199    fn an_anchored_primary_owns_blob_zero_and_its_siblings() {
200        // Blob 0 is the file named verbatim (whatever it is called); every
201        // overflow blob is its sibling named by index. Built through `join` so
202        // the separator is the platform's.
203        let primary = Path::new("out").join("blobs").join("0");
204        assert_eq!(
205            resolve_blob_path(Some(&primary), 0).as_deref(),
206            Some(&*primary.to_string_lossy())
207        );
208        assert_eq!(
209            resolve_blob_path(Some(&primary), 2),
210            Some(
211                Path::new("out")
212                    .join("blobs")
213                    .join("2")
214                    .to_string_lossy()
215                    .into_owned()
216            )
217        );
218
219        // A bare file name hangs its siblings off the working directory.
220        assert_eq!(
221            resolve_blob_path(Some(Path::new("0")), 1).as_deref(),
222            Some("1")
223        );
224
225        // A state tree's `data/` is just a primary named `<tree>/data/0`, so
226        // the overflow blobs land beside it the same way.
227        let tree = crate::store::paths::StateTree::at(Path::new("/proj"));
228        let data = tree.data_dir().join("0");
229        assert_eq!(
230            resolve_blob_path(Some(&data), 3),
231            Some(tree.data_dir().join("3").to_string_lossy().into_owned())
232        );
233    }
234
235    // With nothing anchored there is no layout to resolve against, which is
236    // what turns a blob read into `NoStateRoot` rather than a read of some path
237    // relative to the working directory.
238    #[test]
239    fn without_an_anchor_there_is_no_path() {
240        assert_eq!(resolve_blob_path(None, 0), None);
241        assert_eq!(resolve_blob_path(None, 3), None);
242    }
243
244    #[test]
245    fn format_failures_fold_onto_file_io() {
246        assert_eq!(report("x.cnb", BlobError::BadMagic), CnResult::FileIo);
247        assert_eq!(
248            report("x.cnb", BlobError::ValidityMismatch(99)),
249            CnResult::FileIo
250        );
251    }
252
253    #[test]
254    fn read_cnb_errors_on_a_missing_file() {
255        assert_eq!(
256            read_cnb("/nonexistent/cn/blob/path.cnb"),
257            Err(CnResult::FileIo)
258        );
259    }
260
261    #[test]
262    fn read_payload_section_returns_empty_for_a_short_file() {
263        let dir = tempfile::tempdir().unwrap();
264        let path = dir.path().join("short").to_string_lossy().into_owned();
265        std::fs::write(&path, vec![0u8; HEADER_SIZE - 1]).unwrap();
266        assert!(read_payload_section(&path).unwrap().is_empty());
267    }
268
269    #[test]
270    fn payload_section_start_skips_header_and_meta() {
271        let dir = tempfile::tempdir().unwrap();
272        let path = dir.path().join("0").to_string_lossy().into_owned();
273        let image = encode_cnb(SCHEMA_VERSION, &BlobMeta::default(), b"payloadbytes").unwrap();
274        std::fs::write(&path, &image).unwrap();
275
276        let start = payload_section_start(&path).expect("section start");
277        assert_eq!(&image[start as usize..], b"payloadbytes");
278    }
279
280    #[test]
281    fn payload_section_start_rejects_bad_magic() {
282        let dir = tempfile::tempdir().unwrap();
283        let path = dir.path().join("bad").to_string_lossy().into_owned();
284        std::fs::write(&path, vec![0u8; HEADER_SIZE]).unwrap();
285        assert_eq!(payload_section_start(&path), Err(CnResult::FileIo));
286    }
287
288    #[test]
289    fn load_raw_reads_blob0_eagerly_and_defers_overflow() {
290        use concinnity_core::ecs::{AssetKind, PayloadLocator};
291
292        let dir = tempfile::tempdir().unwrap();
293        let path_for = |idx: u32| {
294            Some(
295                dir.path()
296                    .join(idx.to_string())
297                    .to_string_lossy()
298                    .into_owned(),
299            )
300        };
301
302        // Blob 0: one def whose payload lives in overflow blob 1. The manifest
303        // is derived exactly as cook derives it; `load_raw` trusts its
304        // `max_blob_index` to name the overflow file.
305        let defs = vec![BlobAssetDef {
306            name: None,
307            kind: AssetKind::Component,
308            discriminant: 1,
309            args_bytes: Vec::new(),
310            payload: Some(PayloadLocator {
311                blob_index: 1,
312                offset: 0,
313                len: 8,
314            }),
315        }];
316        let meta = BlobMeta {
317            manifest: WorldManifest::from_records(&defs, &[]),
318            defs,
319            resources: Vec::new(),
320            scene_groups: Vec::new(),
321            mesh_bounds: Vec::new(),
322            physics_budget: None,
323        };
324        std::fs::write(
325            path_for(0).unwrap(),
326            encode_cnb(SCHEMA_VERSION, &meta, b"primary").unwrap(),
327        )
328        .unwrap();
329        std::fs::write(
330            path_for(1).unwrap(),
331            encode_cnb(SCHEMA_VERSION, &BlobMeta::default(), b"overflow").unwrap(),
332        )
333        .unwrap();
334
335        let (meta, mut bd) = load_raw_from(path_for).expect("load");
336        assert_eq!(meta.defs.len(), 1);
337        assert!(meta.resources.is_empty());
338        assert_eq!(meta.manifest.component_counts, vec![(1, 1)]);
339        assert!(bd.disk_backed());
340        // Blob 0 resident, blob 1 deferred until its first read.
341        assert!(bd.is_loaded(0));
342        assert!(!bd.is_loaded(1));
343        let loc = meta.defs[0].payload.clone().unwrap();
344        assert_eq!(bd.read(&loc).expect("overflow read"), b"overflow");
345        assert!(bd.is_loaded(1));
346    }
347
348    // A layout that resolves to nothing is the uninstalled-state-root case, and
349    // it has to name itself rather than folding onto a file-not-found.
350    #[test]
351    fn load_raw_without_a_layout_reports_no_state_root() {
352        assert_eq!(load_raw_from(|_| None).err(), Some(CnResult::NoStateRoot));
353    }
354}
concinnity_host/store/blob/mod.rs

concinnity_host/store/blob/
mod.rs
