concinnity_host/store/blob/
mod.rs

1//! Runtime blob access: the state root's `data/` path layout, the payload
2//! residency store, and all blob file I/O. The concinnity-blob crate owns the
3//! format contract (schema, header, version, bytes <-> metadata) and is
4//! deliberately I/O-free, so every read below is `fs` here plus a pure parse
5//! there. Blob data is read-only at runtime; concinnity-cook writes what
6//! `concinnity_core::blob::encode_cnb` returns.
7use std::fs;
8use std::io::Read;
9use std::path::{Path, PathBuf};
10use std::sync::{Mutex, OnceLock};
11
12pub use concinnity_core::blob::{BLOB_MAGIC, HEADER_SIZE, WorldManifest};
13use concinnity_core::blob::{BlobError, parse_cnb, parse_payload_section_start, payload_section};
14use concinnity_core::result::CnResult;
15
16mod data;
17
18pub use concinnity_core::SCHEMA_VERSION;
19pub use concinnity_core::ecs::{BlobAssetDef, BlobMeta, ResourceRecord};
20pub use data::BlobData;
21
22// Where the blob files live when a host named a primary blob directly, rather
23// than taking the state root's `data/` layout. Holds that file's path; every
24// overflow blob is its sibling named by index.
25fn primary_override() -> &'static Mutex<Option<PathBuf>> {
26    static PRIMARY: OnceLock<Mutex<Option<PathBuf>>> = OnceLock::new();
27    PRIMARY.get_or_init(|| Mutex::new(None))
28}
29
30/// Format a blob file path for a given index. Blob 0 is the primary blob (the
31/// metadata block plus the first payload section); higher indices are overflow
32/// payload blobs, which are always siblings of blob 0. The format crate is
33/// path-agnostic; this layout knowledge stays here.
34///
35/// Blob 0 is `<state dir>/data/0` unless [`load_raw_at`] anchored the layout on
36/// a blob file named directly, in which case that file is blob 0 and its
37/// directory holds the rest. `None` when neither applies: no host installed a
38/// state root and none named a blob, so there is no layout to resolve against.
39pub fn blob_path(index: u32) -> Option<String> {
40    let primary = primary_override().lock().unwrap().clone();
41    resolve_blob_path(
42        primary.as_deref(),
43        super::paths::data_dir().as_deref(),
44        index,
45    )
46}
47
48// Pure resolution split out so the two layouts are unit-testable without the
49// process-global anchor or the process-global state root.
50fn resolve_blob_path(
51    primary: Option<&Path>,
52    data_dir: Option<&Path>,
53    index: u32,
54) -> Option<String> {
55    let path = match primary {
56        Some(p) if index == 0 => p.to_path_buf(),
57        Some(p) => p
58            .parent()
59            .map_or_else(|| PathBuf::from("."), Path::to_path_buf)
60            .join(index.to_string()),
61        None => data_dir?.join(index.to_string()),
62    };
63    Some(path.to_string_lossy().into_owned())
64}
65
66/// Read and deserialize a blob's metadata section (component defs + resource
67/// records). Returns (meta, payload_start_offset).
68pub fn read_cnb(path: &str) -> Result<(BlobMeta, usize), CnResult> {
69    let data = read_file(path)?;
70    parse_cnb::<BlobMeta>(SCHEMA_VERSION, &data).map_err(|e| report(path, e))
71}
72
73/// Byte offset within a blob file at which its payload section begins. Reads
74/// only the header; the disk-backed streaming source uses it to turn a
75/// `PayloadLocator` offset into an absolute file offset.
76/// Used only by the Metal-driven disk-backed streaming source for now
77/// (Vulkan/DirectX streaming catch-up is a follow-up).
78pub fn payload_section_start(path: &str) -> Result<u64, CnResult> {
79    let mut file = fs::File::open(path).map_err(|e| {
80        tracing::error!("Failed to open {}: {}", path, e);
81        CnResult::FileIo
82    })?;
83    let mut header = [0u8; HEADER_SIZE];
84    file.read_exact(&mut header).map_err(|e| {
85        tracing::error!("Failed to read header of {}: {}", path, e);
86        CnResult::FileIo
87    })?;
88    parse_payload_section_start::<BlobMeta>(&header).map_err(|e| report(path, e))
89}
90
91// Read just the payload section of a blob file into memory.
92fn read_payload_section(path: &str) -> Result<Vec<u8>, CnResult> {
93    let data = read_file(path)?;
94    Ok(payload_section(&data).to_vec())
95}
96
97fn read_file(path: &str) -> Result<Vec<u8>, CnResult> {
98    fs::read(path).map_err(|e| {
99        tracing::error!("Failed to read {}: {}", path, e);
100        CnResult::FileIo
101    })
102}
103
104// Log a format failure against the file it came from. The format crate has no
105// path to name, so the diagnostic belongs here.
106fn report(path: &str, e: BlobError) -> CnResult {
107    match e {
108        BlobError::TooShort => tracing::error!("{}: file too short", path),
109        BlobError::BadMagic => tracing::error!("{}: bad magic", path),
110        BlobError::ValidityMismatch(_) => tracing::error!(
111            "{}: world data was built by a different version of the engine",
112            path
113        ),
114        BlobError::TruncatedMeta => tracing::error!("{}: truncated metadata section", path),
115        BlobError::Decode => tracing::error!("{}: failed to deserialize metadata", path),
116        BlobError::TrailingMeta(n) => tracing::error!(
117            "{}: metadata left {} unread bytes; world data was built by a different version of the engine",
118            path,
119            n
120        ),
121        BlobError::Encode => tracing::error!("{}: failed to serialize metadata", path),
122    }
123    CnResult::FileIo
124}
125
126/// Load the blob file at `primary` and the payload store around it, anchoring
127/// the process's blob layout on it: that file is blob 0 and its siblings named
128/// by index are the overflow payload blobs, so a world written to `data/0`
129/// reads `data/1`, `data/2`, ... beside it. The anchor outlives the call
130/// because payloads stream off disk long after startup.
131pub fn load_raw_at(primary: &Path) -> Result<(BlobMeta, BlobData), CnResult> {
132    *primary_override().lock().unwrap() = Some(primary.to_path_buf());
133    load_raw()
134}
135
136/// Load the primary blob's metadata and the `BlobData` payload store from the
137/// state root's `data/` layout, without resolving defs into runtime `Asset`s
138/// (that resolution depends on the client runtime registry, so it lives in the
139/// client `blob::load` shim).
140///
141/// Only blob 0's payload section is read here; overflow blobs (named by the
142/// manifest's `max_blob_index`) start unloaded and `BlobData::read()` pulls
143/// each from disk the first time a locator needs it.
144pub fn load_raw() -> Result<(BlobMeta, BlobData), CnResult> {
145    load_raw_from(blob_path)
146}
147
148// `load_raw` against an injected layout, so the eager/deferred split can be
149// exercised without the process-global data-dir anchor.
150fn load_raw_from(
151    blob_path: impl Fn(u32) -> Option<String>,
152) -> Result<(BlobMeta, BlobData), CnResult> {
153    let (meta, _payload_start) = read_cnb(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
154
155    // Cook derives the manifest from the very streams it summarizes, so a
156    // mismatch means a corrupt or hand-edited blob.
157    debug_assert_eq!(
158        meta.manifest,
159        WorldManifest::from_records(&meta.defs, &meta.resources),
160        "blob manifest does not match its record streams"
161    );
162
163    let blob0_payload = read_payload_section(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
164    tracing::debug!("Loaded blob 0 payload ({} bytes)", blob0_payload.len());
165    let overflow_paths = (1..=meta.manifest.max_blob_index)
166        .map(|i| blob_path(i).ok_or(CnResult::NoStateRoot))
167        .collect::<Result<Vec<_>, _>>()?;
168
169    let blob_data = BlobData::from_blob_files(blob0_payload, overflow_paths);
170    Ok((meta, blob_data))
171}
172
173/// Number of Texture resource records in the primary blob's metadata, read
174/// without loading any payload. This is the compiled world's texture-table
175/// length; `cn export` uses it to precompile the built-in shaders whose bindless
176/// texture pool is sized per world.
177pub fn texture_resource_count() -> Result<usize, CnResult> {
178    let (meta, _) = read_cnb(&blob_path(0).ok_or(CnResult::NoStateRoot)?)?;
179    let tag = concinnity_core::ecs::ResourceKind::Texture as u8;
180    Ok(meta
181        .resources
182        .iter()
183        .filter(|r| r.resource_kind == tag)
184        .count())
185}
186
187/// Load defs without resolving (for callers that apply overlays first)
188pub fn load_defs() -> Result<Vec<BlobAssetDef>, CnResult> {
189    read_cnb(&blob_path(0).ok_or(CnResult::NoStateRoot)?).map(|(meta, _)| meta.defs)
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195    use concinnity_core::blob::encode_cnb;
196
197    #[test]
198    fn an_anchored_primary_owns_blob_zero_and_its_siblings() {
199        // Blob 0 is the file named verbatim (whatever it is called); every
200        // overflow blob is its sibling named by index. Built through `join` so
201        // the separator is the platform's.
202        let primary = Path::new("out").join("blobs").join("0");
203        assert_eq!(
204            resolve_blob_path(Some(&primary), None, 0).as_deref(),
205            Some(&*primary.to_string_lossy())
206        );
207        assert_eq!(
208            resolve_blob_path(Some(&primary), None, 2),
209            Some(
210                Path::new("out")
211                    .join("blobs")
212                    .join("2")
213                    .to_string_lossy()
214                    .into_owned()
215            )
216        );
217
218        // A bare file name hangs its siblings off the working directory.
219        assert_eq!(
220            resolve_blob_path(Some(Path::new("0")), None, 1).as_deref(),
221            Some("1")
222        );
223    }
224
225    #[test]
226    fn without_an_anchor_blobs_sit_under_the_data_dir() {
227        let data = Path::new("/proj").join("data");
228        assert_eq!(
229            resolve_blob_path(None, Some(&data), 3),
230            Some(data.join("3").to_string_lossy().into_owned())
231        );
232    }
233
234    // With neither an anchor nor a state root there is no layout to resolve
235    // against, which is what turns a blob read into `NoStateRoot` rather than a
236    // read of some path relative to the working directory.
237    #[test]
238    fn without_an_anchor_or_a_state_root_there_is_no_path() {
239        assert_eq!(resolve_blob_path(None, None, 0), None);
240        assert_eq!(resolve_blob_path(None, None, 3), None);
241    }
242
243    #[test]
244    fn format_failures_fold_onto_file_io() {
245        assert_eq!(report("x.cnb", BlobError::BadMagic), CnResult::FileIo);
246        assert_eq!(
247            report("x.cnb", BlobError::ValidityMismatch(99)),
248            CnResult::FileIo
249        );
250    }
251
252    #[test]
253    fn read_cnb_errors_on_a_missing_file() {
254        assert_eq!(
255            read_cnb("/nonexistent/cn/blob/path.cnb"),
256            Err(CnResult::FileIo)
257        );
258    }
259
260    #[test]
261    fn read_payload_section_returns_empty_for_a_short_file() {
262        let dir = tempfile::tempdir().unwrap();
263        let path = dir.path().join("short").to_string_lossy().into_owned();
264        std::fs::write(&path, vec![0u8; HEADER_SIZE - 1]).unwrap();
265        assert!(read_payload_section(&path).unwrap().is_empty());
266    }
267
268    #[test]
269    fn payload_section_start_skips_header_and_meta() {
270        let dir = tempfile::tempdir().unwrap();
271        let path = dir.path().join("0").to_string_lossy().into_owned();
272        let image = encode_cnb(SCHEMA_VERSION, &BlobMeta::default(), b"payloadbytes").unwrap();
273        std::fs::write(&path, &image).unwrap();
274
275        let start = payload_section_start(&path).expect("section start");
276        assert_eq!(&image[start as usize..], b"payloadbytes");
277    }
278
279    #[test]
280    fn payload_section_start_rejects_bad_magic() {
281        let dir = tempfile::tempdir().unwrap();
282        let path = dir.path().join("bad").to_string_lossy().into_owned();
283        std::fs::write(&path, vec![0u8; HEADER_SIZE]).unwrap();
284        assert_eq!(payload_section_start(&path), Err(CnResult::FileIo));
285    }
286
287    #[test]
288    fn load_raw_reads_blob0_eagerly_and_defers_overflow() {
289        use concinnity_core::ecs::{AssetKind, PayloadLocator};
290
291        let dir = tempfile::tempdir().unwrap();
292        let path_for = |idx: u32| {
293            Some(
294                dir.path()
295                    .join(idx.to_string())
296                    .to_string_lossy()
297                    .into_owned(),
298            )
299        };
300
301        // Blob 0: one def whose payload lives in overflow blob 1. The manifest
302        // is derived exactly as cook derives it; `load_raw` trusts its
303        // `max_blob_index` to name the overflow file.
304        let defs = vec![BlobAssetDef {
305            name: None,
306            kind: AssetKind::Component,
307            discriminant: 1,
308            args_bytes: Vec::new(),
309            payload: Some(PayloadLocator {
310                blob_index: 1,
311                offset: 0,
312                len: 8,
313            }),
314        }];
315        let meta = BlobMeta {
316            manifest: WorldManifest::from_records(&defs, &[]),
317            defs,
318            resources: Vec::new(),
319            scene_groups: Vec::new(),
320            mesh_bounds: Vec::new(),
321            physics_budget: None,
322        };
323        std::fs::write(
324            path_for(0).unwrap(),
325            encode_cnb(SCHEMA_VERSION, &meta, b"primary").unwrap(),
326        )
327        .unwrap();
328        std::fs::write(
329            path_for(1).unwrap(),
330            encode_cnb(SCHEMA_VERSION, &BlobMeta::default(), b"overflow").unwrap(),
331        )
332        .unwrap();
333
334        let (meta, mut bd) = load_raw_from(path_for).expect("load");
335        assert_eq!(meta.defs.len(), 1);
336        assert!(meta.resources.is_empty());
337        assert_eq!(meta.manifest.component_counts, vec![(1, 1)]);
338        assert!(bd.disk_backed());
339        // Blob 0 resident, blob 1 deferred until its first read.
340        assert!(bd.is_loaded(0));
341        assert!(!bd.is_loaded(1));
342        let loc = meta.defs[0].payload.clone().unwrap();
343        assert_eq!(bd.read(&loc).expect("overflow read"), b"overflow");
344        assert!(bd.is_loaded(1));
345    }
346
347    // A layout that resolves to nothing is the uninstalled-state-root case, and
348    // it has to name itself rather than folding onto a file-not-found.
349    #[test]
350    fn load_raw_without_a_layout_reports_no_state_root() {
351        assert_eq!(load_raw_from(|_| None).err(), Some(CnResult::NoStateRoot));
352    }
353}
concinnity_host/store/blob/mod.rs

concinnity_host/store/blob/
mod.rs
