vole_document/container/
descriptor.rs

1//! High-level `.voldoc` descriptor: serialize and parse.
2//!
3//! A descriptor binds a source-format class, a universe declaration, raw byte
4//! objects, a reconstruction program, and a whole-source integrity manifest
5//! into one framed container. Parsing is structural and bounded; it does not
6//! materialize bytes. Materialization is a separate, explicit step.
7
8use crate::EXACTNESS_PROFILE_EXACT_BYTES;
9use crate::accounting::CostBreakdown;
10use crate::container::header::{HEADER_LEN, Header, MAGIC};
11use crate::container::record::{RECORD_OVERHEAD, RecordReader, RecordTag};
12use crate::dra::Program;
13use crate::entropy::codec::EntropyChannelDescriptor;
14use crate::entropy::model::EntropyModel;
15use crate::error::{Error, Result};
16use crate::integrity::sha256;
17use crate::limits::Limits;
18
19/// The Phase-2 reconstruction universe declaration.
20///
21/// Changing any opcode, coder, limit semantic, or adapter meaning requires a
22/// new universe string. The `universe_id` in the header is the first 16 bytes
23/// of SHA-256 over this string.
24pub const UNIVERSE: &str = "vole-document;universe;phase-3;exact-bytes;dra-2;opaque+entropy+pdf";
25
26/// First 16 bytes of SHA-256 over a universe declaration string.
27pub fn universe_id_from_str(universe: &str) -> [u8; 16] {
28    let full = sha256(universe.as_bytes());
29    let mut id = [0u8; 16];
30    id.copy_from_slice(&full[0..16]);
31    id
32}
33
34/// The in-memory model of a `.voldoc` descriptor.
35#[derive(Debug, Clone, PartialEq, Eq)]
36pub struct Descriptor {
37    /// Universe declaration string.
38    pub universe: String,
39    /// Source-format class selector.
40    pub source_format: u8,
41    /// Human-readable basis for the format decision (provenance, not trust).
42    pub format_basis: String,
43    /// Canonical entropy models referenced by channels.
44    pub models: Vec<EntropyModel>,
45    /// Typed entropy channels referenced by the program.
46    pub channels: Vec<EntropyChannelDescriptor>,
47    /// Raw byte objects referenced by the program.
48    pub objects: Vec<Vec<u8>>,
49    /// The reconstruction program.
50    pub program: Program,
51    /// SHA-256 of the exact reconstructed source.
52    pub source_sha256: [u8; 32],
53    /// Exact reconstructed source length.
54    pub source_len: u64,
55}
56
57/// A parsed descriptor plus its physical cost breakdown.
58#[derive(Debug, Clone)]
59pub struct ParsedDescriptor {
60    /// The descriptor model.
61    pub descriptor: Descriptor,
62    /// Physical byte attribution of the serialized form.
63    pub cost: CostBreakdown,
64    /// Encode the universe identifier that was validated against the header.
65    pub universe_id: [u8; 16],
66}
67
68impl Descriptor {
69    /// The header that this descriptor serializes to.
70    pub fn header(&self) -> Header {
71        Header::new(
72            universe_id_from_str(&self.universe),
73            self.source_len,
74            EXACTNESS_PROFILE_EXACT_BYTES,
75            self.source_format,
76        )
77    }
78
79    /// Serialize to a complete `.voldoc` byte sequence plus cost attribution.
80    pub fn serialize(&self) -> Result<(Vec<u8>, CostBreakdown)> {
81        let mut cost = CostBreakdown {
82            header: HEADER_LEN as u64,
83            ..Default::default()
84        };
85        let mut out = Vec::new();
86
87        let header = self.header();
88        out.extend_from_slice(&header.encode());
89
90        let mut records: u64 = 0;
91        let mut write = |out: &mut Vec<u8>, tag: RecordTag, payload: &[u8]| -> Result<()> {
92            crate::container::record::write_record(out, tag as u8, 0, payload)?;
93            records += 1;
94            Ok(())
95        };
96
97        // UNIVERSE
98        write(&mut out, RecordTag::Universe, self.universe.as_bytes())?;
99        cost.universe = self.universe.len() as u64;
100
101        // FORMAT: [class u8][basis_len u32 LE][basis bytes]
102        let basis = self.format_basis.as_bytes();
103        let basis_len = u32::try_from(basis.len())
104            .map_err(|_| Error::resource_limit("format basis too long"))?;
105        let mut fmt = Vec::with_capacity(5 + basis.len());
106        fmt.push(self.source_format);
107        fmt.extend_from_slice(&basis_len.to_le_bytes());
108        fmt.extend_from_slice(basis);
109        write(&mut out, RecordTag::Format, &fmt)?;
110        cost.format = fmt.len() as u64;
111
112        // MODELS
113        for model in &self.models {
114            let encoded = model.encode()?;
115            write(&mut out, RecordTag::Model, &encoded)?;
116            cost.models += encoded.len() as u64;
117        }
118
119        // ENTROPY CHANNELS
120        for channel in &self.channels {
121            let encoded = channel.encode()?;
122            write(&mut out, RecordTag::EntropyChannel, &encoded)?;
123            cost.entropy_payload += encoded.len() as u64;
124        }
125
126        // OBJECTS
127        for obj in &self.objects {
128            write(&mut out, RecordTag::Object, obj)?;
129            cost.objects += obj.len() as u64;
130        }
131
132        // GRAPH
133        let graph = self.program.encode()?;
134        write(&mut out, RecordTag::Graph, &graph)?;
135        cost.graph = graph.len() as u64;
136
137        // INTEGRITY: [sha256 32][source_len u64 LE]
138        let mut integ = Vec::with_capacity(40);
139        integ.extend_from_slice(&self.source_sha256);
140        integ.extend_from_slice(&self.source_len.to_le_bytes());
141        write(&mut out, RecordTag::Integrity, &integ)?;
142        cost.integrity = integ.len() as u64;
143
144        // TRAILER: [record_count u32][payload_bytes u64][MAGIC 8]
145        // record_count includes the trailer itself.
146        let total_records =
147            u32::try_from(records + 1).map_err(|_| Error::resource_limit("too many records"))?;
148        let payload_bytes = (out.len() - HEADER_LEN) as u64;
149        let mut trailer = Vec::with_capacity(20);
150        trailer.extend_from_slice(&total_records.to_le_bytes());
151        trailer.extend_from_slice(&payload_bytes.to_le_bytes());
152        trailer.extend_from_slice(&MAGIC);
153        crate::container::record::write_record(&mut out, RecordTag::Trailer as u8, 0, &trailer)?;
154        cost.trailer = trailer.len() as u64;
155
156        // Framing overhead for every record after the fixed header.
157        cost.record_framing = RECORD_OVERHEAD as u64 * total_records as u64;
158
159        debug_assert_eq!(cost.total(), out.len() as u64);
160        Ok((out, cost))
161    }
162
163    /// Parse a complete `.voldoc` byte sequence with structural validation.
164    ///
165    /// This validates framing, universe identity, version, mandatory features,
166    /// record presence, the coverage certificate, and declared length. It does
167    /// **not** materialize or hash the reconstructed source; call
168    /// [`crate::materialize::materialize`] for that.
169    pub fn parse(bytes: &[u8], limits: Limits) -> Result<ParsedDescriptor> {
170        if bytes.len() as u64 > limits.max_input_bytes {
171            return Err(Error::resource_limit(
172                "input exceeds configured input limit",
173            ));
174        }
175        let header = Header::decode(bytes)?;
176        if !header.source_format_supported() {
177            return Err(Error::unsupported_feature(format!(
178                "source format class {} has no adapter in this build",
179                header.source_format
180            )));
181        }
182
183        let mut cost = CostBreakdown {
184            header: HEADER_LEN as u64,
185            ..Default::default()
186        };
187
188        let mut reader = RecordReader::new(bytes, HEADER_LEN, limits);
189        let mut universe: Option<String> = None;
190        let mut format: Option<(u8, String)> = None;
191        let mut models: Vec<EntropyModel> = Vec::new();
192        let mut channels: Vec<EntropyChannelDescriptor> = Vec::new();
193        let mut objects: Vec<Vec<u8>> = Vec::new();
194        let mut program: Option<Program> = None;
195        let mut source_sha256: Option<[u8; 32]> = None;
196        let mut source_len: Option<u64> = None;
197        let mut saw_trailer = false;
198        let mut trailer_record_count: Option<u32> = None;
199        let mut records_seen: u32 = 0;
200
201        while let Some(rec) = reader.next_record()? {
202            records_seen += 1;
203            if saw_trailer {
204                return Err(Error::invalid_container("record found after TRAILER"));
205            }
206            match RecordTag::from_u8(rec.tag) {
207                Some(RecordTag::Universe) => {
208                    if universe.is_some() {
209                        return Err(Error::invalid_container("duplicate UNIVERSE record"));
210                    }
211                    let payload_len = rec.payload.len();
212                    let s = String::from_utf8(rec.payload)
213                        .map_err(|_| Error::invalid_container("universe is not valid UTF-8"))?;
214                    if universe_id_from_str(&s) != header.universe_id {
215                        return Err(Error::invalid_container(
216                            "universe declaration does not match its header identifier",
217                        ));
218                    }
219                    universe = Some(s);
220                    cost.universe = payload_len as u64;
221                }
222                Some(RecordTag::Format) => {
223                    if format.is_some() {
224                        return Err(Error::invalid_container("duplicate FORMAT record"));
225                    }
226                    if rec.payload.len() < 5 {
227                        return Err(Error::invalid_container("truncated FORMAT payload"));
228                    }
229                    let class = rec.payload[0];
230                    let blen = u32::from_le_bytes([
231                        rec.payload[1],
232                        rec.payload[2],
233                        rec.payload[3],
234                        rec.payload[4],
235                    ]);
236                    let blen = blen as usize;
237                    if rec.payload.len() != 5 + blen {
238                        return Err(Error::invalid_container("FORMAT payload length mismatch"));
239                    }
240                    let basis = String::from_utf8(rec.payload[5..].to_vec())
241                        .map_err(|_| Error::invalid_container("format basis is not UTF-8"))?;
242                    if class != header.source_format {
243                        return Err(Error::invalid_container(
244                            "FORMAT class disagrees with header source_format",
245                        ));
246                    }
247                    format = Some((class, basis));
248                    cost.format = rec.payload.len() as u64;
249                }
250                Some(RecordTag::Object) => {
251                    if objects.len() as u32 >= limits.max_object_count {
252                        return Err(Error::resource_limit("object count limit exceeded"));
253                    }
254                    cost.objects += rec.payload.len() as u64;
255                    objects.push(rec.payload);
256                }
257                Some(RecordTag::Model) => {
258                    if models.len() as u32 >= limits.max_model_count {
259                        return Err(Error::resource_limit("entropy model count limit exceeded"));
260                    }
261                    if rec.payload.len() as u32 > limits.max_entropy_model_bytes {
262                        return Err(Error::resource_limit(format!(
263                            "entropy model payload {} exceeds limit {}",
264                            rec.payload.len(),
265                            limits.max_entropy_model_bytes
266                        )));
267                    }
268                    let model = EntropyModel::decode(&rec.payload)?;
269                    cost.models += rec.payload.len() as u64;
270                    models.push(model);
271                }
272                Some(RecordTag::EntropyChannel) => {
273                    if channels.len() as u32 >= limits.max_channel_count {
274                        return Err(Error::resource_limit(
275                            "entropy channel count limit exceeded",
276                        ));
277                    }
278                    let channel = EntropyChannelDescriptor::decode(&rec.payload, limits)?;
279                    cost.entropy_payload += rec.payload.len() as u64;
280                    channels.push(channel);
281                }
282                Some(RecordTag::Graph) => {
283                    if program.is_some() {
284                        return Err(Error::invalid_container("duplicate GRAPH record"));
285                    }
286                    let p = Program::decode(&rec.payload, limits)?;
287                    cost.graph = rec.payload.len() as u64;
288                    program = Some(p);
289                }
290                Some(RecordTag::Integrity) => {
291                    if source_sha256.is_some() {
292                        return Err(Error::invalid_container("duplicate INTEGRITY record"));
293                    }
294                    if rec.payload.len() != 40 {
295                        return Err(Error::invalid_container(
296                            "INTEGRITY payload must be 40 bytes",
297                        ));
298                    }
299                    let mut sha = [0u8; 32];
300                    sha.copy_from_slice(&rec.payload[0..32]);
301                    let len = u64::from_le_bytes([
302                        rec.payload[32],
303                        rec.payload[33],
304                        rec.payload[34],
305                        rec.payload[35],
306                        rec.payload[36],
307                        rec.payload[37],
308                        rec.payload[38],
309                        rec.payload[39],
310                    ]);
311                    source_sha256 = Some(sha);
312                    source_len = Some(len);
313                    cost.integrity = rec.payload.len() as u64;
314                }
315                Some(RecordTag::Trailer) => {
316                    if rec.payload.len() != 20 {
317                        return Err(Error::invalid_container("TRAILER payload must be 20 bytes"));
318                    }
319                    if rec.payload[12..20] != MAGIC {
320                        return Err(Error::invalid_container("TRAILER magic mismatch"));
321                    }
322                    trailer_record_count = Some(u32::from_le_bytes([
323                        rec.payload[0],
324                        rec.payload[1],
325                        rec.payload[2],
326                        rec.payload[3],
327                    ]));
328                    cost.trailer = rec.payload.len() as u64;
329                    saw_trailer = true;
330                }
331                // Phase 2+ mandatory records have no meaning in this universe.
332                Some(RecordTag::Residual)
333                | Some(RecordTag::Checkpoint)
334                | Some(RecordTag::Index)
335                | Some(RecordTag::ExternalRef) => {
336                    if rec.is_optional() {
337                        // Explicitly optional and unknown to this universe: skip.
338                    } else {
339                        return Err(Error::unsupported_feature(format!(
340                            "record class {} requires a universe this build does not implement",
341                            rec.tag
342                        )));
343                    }
344                }
345                None => {
346                    if rec.is_optional() {
347                        // Forward-compatible optional record: skip.
348                    } else {
349                        return Err(Error::unsupported_feature(format!(
350                            "unknown mandatory record tag {:#04x}",
351                            rec.tag
352                        )));
353                    }
354                }
355            }
356        }
357
358        let universe =
359            universe.ok_or_else(|| Error::invalid_container("missing UNIVERSE record"))?;
360        let (class, basis) =
361            format.ok_or_else(|| Error::invalid_container("missing FORMAT record"))?;
362        let program = program.ok_or_else(|| Error::invalid_container("missing GRAPH record"))?;
363        let source_sha256 =
364            source_sha256.ok_or_else(|| Error::invalid_container("missing INTEGRITY record"))?;
365        let source_len =
366            source_len.ok_or_else(|| Error::invalid_container("missing INTEGRITY record"))?;
367        if !saw_trailer {
368            return Err(Error::invalid_container("missing TRAILER record"));
369        }
370        if let Some(n) = trailer_record_count
371            && n != records_seen
372        {
373            return Err(Error::invalid_container(format!(
374                "TRAILER declares {n} records but {records_seen} were read"
375            )));
376        }
377        if source_len != header.declared_source_len {
378            return Err(Error::integrity_mismatch(format!(
379                "INTEGRITY length {source_len} disagrees with header {}",
380                header.declared_source_len
381            )));
382        }
383
384        // Cross-validate every channel against the model it references. A
385        // channel may not name a missing model, and its declared scale must
386        // agree with that model.
387        for (i, channel) in channels.iter().enumerate() {
388            let model = models.get(channel.model_id as usize).ok_or_else(|| {
389                Error::invalid_model(format!(
390                    "entropy channel {i} references missing model {}",
391                    channel.model_id
392                ))
393            })?;
394            if channel.scale_bits != model.scale_bits {
395                return Err(Error::invalid_model(format!(
396                    "entropy channel {i} scale_bits {} disagrees with model {} scale_bits {}",
397                    channel.scale_bits, channel.model_id, model.scale_bits
398                )));
399            }
400        }
401
402        // Coverage certificate: every source byte has exactly one authority and
403        // the program's predicted length equals the declared length.
404        let object_lens: Vec<u64> = objects.iter().map(|o| o.len() as u64).collect();
405        let channel_lens: Vec<u64> = channels.iter().map(|c| c.decoded_length).collect();
406        let (predicted, coverage) = program.analyze(&object_lens, &channel_lens, limits)?;
407        if predicted != source_len {
408            return Err(Error::coverage_violation(format!(
409                "reconstruction program predicts {predicted} bytes but {source_len} were declared"
410            )));
411        }
412        coverage.validate(source_len)?;
413
414        cost.record_framing = RECORD_OVERHEAD as u64 * records_seen as u64;
415
416        Ok(ParsedDescriptor {
417            descriptor: Descriptor {
418                universe,
419                source_format: class,
420                format_basis: basis,
421                models,
422                channels,
423                objects,
424                program,
425                source_sha256,
426                source_len,
427            },
428            cost,
429            universe_id: header.universe_id,
430        })
431    }
432}
433
434#[cfg(test)]
435mod tests {
436    use super::*;
437    use crate::SOURCE_FORMAT_OPAQUE;
438    use crate::dra::Op;
439    use crate::integrity::sha256;
440
441    fn sample(source: &[u8]) -> Descriptor {
442        Descriptor {
443            universe: UNIVERSE.to_string(),
444            source_format: SOURCE_FORMAT_OPAQUE,
445            format_basis: "opaque:test".to_string(),
446            models: vec![],
447            channels: vec![],
448            objects: vec![source.to_vec()],
449            program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
450            source_sha256: sha256(source),
451            source_len: source.len() as u64,
452        }
453    }
454
455    fn channel(model_id: u32, scale_bits: u8, decoded_length: u64) -> EntropyChannelDescriptor {
456        EntropyChannelDescriptor {
457            coder: crate::entropy::codec::CODER_ORDER0_BYTE_RANS,
458            coder_version: crate::entropy::codec::CODER_VERSION_1,
459            scale_bits,
460            lane_count: 1,
461            model_id,
462            symbol_count: decoded_length,
463            decoded_length,
464            initial_state: 1,
465            payload: vec![0u8; 4],
466        }
467    }
468
469    #[test]
470    fn model_and_channel_roundtrip() {
471        let payload = b"channel bytes";
472        let mut d = sample(payload);
473        d.models = vec![EntropyModel::uniform(8).unwrap()];
474        d.channels = vec![channel(0, 8, payload.len() as u64)];
475        d.program = Program::new(vec![Op::DecodeChannel { channel_id: 0 }]);
476        let (bytes, cost) = d.serialize().unwrap();
477        assert_eq!(cost.total(), bytes.len() as u64);
478        assert!(cost.models > 0);
479        assert!(cost.entropy_payload > 0);
480        let parsed = Descriptor::parse(&bytes, Limits::DEFAULT).unwrap();
481        assert_eq!(parsed.descriptor, d);
482        assert_eq!(parsed.cost.total(), bytes.len() as u64);
483    }
484
485    #[test]
486    fn channel_with_missing_model_rejected() {
487        let mut d = sample(b"abc");
488        d.program = Program::new(vec![Op::DecodeChannel { channel_id: 0 }]);
489        d.channels = vec![channel(3, 8, 3)];
490        let (bytes, _) = d.serialize().unwrap();
491        let e = Descriptor::parse(&bytes, Limits::DEFAULT).unwrap_err();
492        assert_eq!(e.class(), crate::ErrorClass::InvalidModel);
493    }
494
495    #[test]
496    fn channel_scale_mismatch_rejected() {
497        let mut d = sample(b"abc");
498        d.models = vec![EntropyModel::uniform(8).unwrap()];
499        d.program = Program::new(vec![Op::DecodeChannel { channel_id: 0 }]);
500        d.channels = vec![channel(0, 12, 3)];
501        let (bytes, _) = d.serialize().unwrap();
502        let e = Descriptor::parse(&bytes, Limits::DEFAULT).unwrap_err();
503        assert_eq!(e.class(), crate::ErrorClass::InvalidModel);
504    }
505
506    #[test]
507    fn serialize_parse_roundtrip() {
508        let d = sample(b"hello, exact world");
509        let (bytes, cost) = d.serialize().unwrap();
510        assert_eq!(cost.total(), bytes.len() as u64);
511        let parsed = Descriptor::parse(&bytes, Limits::DEFAULT).unwrap();
512        assert_eq!(parsed.descriptor, d);
513        assert_eq!(parsed.cost.total(), bytes.len() as u64);
514    }
515
516    #[test]
517    fn trailing_bytes_after_trailer_rejected() {
518        let d = sample(b"abc");
519        let (mut bytes, _) = d.serialize().unwrap();
520        bytes.push(0);
521        let e = Descriptor::parse(&bytes, Limits::DEFAULT).unwrap_err();
522        assert_eq!(e.class(), crate::ErrorClass::InvalidContainer);
523    }
524
525    #[test]
526    fn declared_length_mismatch_rejected() {
527        // Build a descriptor whose declared length disagrees with the program.
528        let mut d = sample(b"abcdef");
529        d.source_len = 5;
530        let (bytes, _) = d.serialize().unwrap();
531        let e = Descriptor::parse(&bytes, Limits::DEFAULT).unwrap_err();
532        assert_eq!(e.class(), crate::ErrorClass::CoverageViolation);
533    }
534}
vole_document/container/descriptor.rs

vole_document/container/
descriptor.rs
