vyre_foundation/serial/wire/encode/
scan_database_header.rs

1//! Versioned scan database header framing.
2//!
3//! This is not a `VIR0` Program payload. It is the canonical cache/evidence
4//! header for serialized scan databases that reference compiled pattern sets,
5//! table sections, and unsupported construct diagnostics.
6
7use super::WireEncodeErr;
8use crate::serial::wire::framing::{put_len_u32, put_string, put_u32, put_u8};
9use crate::serial::wire::Reader;
10use serde::{Deserialize, Serialize};
11
12/// Four-byte magic identifying a versioned scan database header (`VSDH`).
13pub const SCAN_DATABASE_HEADER_MAGIC: &[u8; 4] = b"VSDH";
14/// Current scan database header wire version.
15pub const SCAN_DATABASE_HEADER_VERSION: u32 = 1;
16/// Upper bound on table sections accepted from an untrusted header.
17pub const MAX_SCAN_DATABASE_SECTIONS: usize = 4_096;
18/// Upper bound on unsupported-feature diagnostics accepted from a header.
19pub const MAX_SCAN_DATABASE_UNSUPPORTED_FEATURES: usize = 4_096;
20
21/// Scan execution mode the database was compiled for.
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
23pub enum ScanDatabaseMode {
24    /// Whole-buffer block scanning.
25    Block,
26    /// Incremental streaming scan with carried state.
27    Streaming,
28    /// Vectored scan over multiple discontiguous buffers.
29    Vectored,
30}
31
32impl ScanDatabaseMode {
33    const fn tag(self) -> u8 {
34        match self {
35            Self::Block => 1,
36            Self::Streaming => 2,
37            Self::Vectored => 3,
38        }
39    }
40
41    fn from_tag(tag: u8) -> Result<Self, String> {
42        match tag {
43            1 => Ok(Self::Block),
44            2 => Ok(Self::Streaming),
45            3 => Ok(Self::Vectored),
46            _ => Err(format!(
47                "scan database mode tag {tag} is unsupported. Fix: recompile the scan database with a compatible Vyre scan compiler."
48            )),
49        }
50    }
51}
52
53/// Kind of payload a table section carries.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
55pub enum ScanDatabaseSectionKind {
56    /// Compiled literal-set table.
57    LiteralTable,
58    /// Compiled automata (DFA/NFA) transition table.
59    AutomataTable,
60    /// Verifier-only regex fragments for constructs the engine cannot match.
61    VerifierFragments,
62    /// Output/result layout descriptor.
63    OutputLayout,
64    /// Carried streaming-scan state.
65    StreamingState,
66    /// Relation seed table for cross-pattern relations.
67    RelationSeeds,
68}
69
70impl ScanDatabaseSectionKind {
71    const fn tag(self) -> u8 {
72        match self {
73            Self::LiteralTable => 1,
74            Self::AutomataTable => 2,
75            Self::VerifierFragments => 3,
76            Self::OutputLayout => 4,
77            Self::StreamingState => 5,
78            Self::RelationSeeds => 6,
79        }
80    }
81
82    fn from_tag(tag: u8) -> Result<Self, String> {
83        match tag {
84            1 => Ok(Self::LiteralTable),
85            2 => Ok(Self::AutomataTable),
86            3 => Ok(Self::VerifierFragments),
87            4 => Ok(Self::OutputLayout),
88            5 => Ok(Self::StreamingState),
89            6 => Ok(Self::RelationSeeds),
90            _ => Err(format!(
91                "scan database section tag {tag} is unsupported. Fix: recompile the scan database with a compatible Vyre scan compiler."
92            )),
93        }
94    }
95}
96
97/// Locator and integrity digest for one table section in the database body.
98#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
99pub struct ScanDatabaseSectionHeader {
100    /// Kind of payload this section carries.
101    pub kind: ScanDatabaseSectionKind,
102    /// Byte offset of the section payload from the start of the database body.
103    pub offset: u64,
104    /// Byte length of the section payload.
105    pub byte_len: u64,
106    /// Integrity digest over the section payload bytes.
107    pub section_digest: u64,
108}
109
110/// A construct the compiler could not lower, recorded for verifier handoff.
111#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
112pub struct UnsupportedScanFeature {
113    /// Index of the pattern that used the unsupported construct.
114    pub pattern_index: u32,
115    /// Human-readable description of the unsupported feature.
116    pub feature: String,
117}
118
119/// How a reader may consume a database given its unsupported constructs.
120#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
121pub enum ScanDatabaseReaderCompatibility {
122    /// Fully consumable by the engine alone.
123    Compatible,
124    /// Consumable only with a verifier for the unsupported fragments.
125    RequiresVerifier,
126    /// Not consumable by this reader at all.
127    Incompatible,
128}
129
130impl ScanDatabaseReaderCompatibility {
131    const fn tag(self) -> u8 {
132        match self {
133            Self::Compatible => 1,
134            Self::RequiresVerifier => 2,
135            Self::Incompatible => 3,
136        }
137    }
138
139    fn from_tag(tag: u8) -> Result<Self, String> {
140        match tag {
141            1 => Ok(Self::Compatible),
142            2 => Ok(Self::RequiresVerifier),
143            3 => Ok(Self::Incompatible),
144            _ => Err(format!(
145                "scan database reader compatibility tag {tag} is unsupported. Fix: rebuild the scan database cache with a compatible Vyre scan compiler."
146            )),
147        }
148    }
149}
150
151/// Construct-tier and dialect compatibility fingerprint for a database.
152#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
153pub struct ScanDatabaseCompatibilityRecord {
154    /// Digest of the construct-tier matrix the database was compiled against.
155    pub construct_tier_digest: u64,
156    /// Digest of the regex dialect lattice the database was compiled against.
157    pub dialect_digest: u64,
158    /// Reader compatibility class for this database.
159    pub reader_compatibility: ScanDatabaseReaderCompatibility,
160}
161
162/// Versioned, self-describing header for a serialized scan database.
163#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
164pub struct ScanDatabaseHeader {
165    /// 32-byte digest of the compiled pattern set.
166    pub pattern_set_digest: [u8; 32],
167    /// Version string of the compiler that produced the database.
168    pub compiler_version: String,
169    /// Scan mode the database was compiled for.
170    pub mode: ScanDatabaseMode,
171    /// Locators for the table sections in the database body.
172    pub table_sections: Vec<ScanDatabaseSectionHeader>,
173    /// Constructs that require verifier handoff.
174    pub unsupported_features: Vec<UnsupportedScanFeature>,
175    /// Construct-tier and dialect compatibility fingerprint.
176    pub compatibility: ScanDatabaseCompatibilityRecord,
177}
178
179impl ScanDatabaseHeader {
180    /// Number of table sections in the database body.
181    #[must_use]
182    pub fn section_count(&self) -> usize {
183        self.table_sections.len()
184    }
185
186    /// Number of unsupported-feature diagnostics recorded in the header.
187    #[must_use]
188    pub fn unsupported_feature_count(&self) -> usize {
189        self.unsupported_features.len()
190    }
191
192    /// Validate this header against the consumer's required compiler version
193    /// and scan mode before any table payload is trusted.
194    ///
195    /// # Errors
196    ///
197    /// Returns an actionable `Fix:` diagnostic when compiler version or mode
198    /// differs from the expected cache key.
199    pub fn validate_compatible(
200        &self,
201        expected_compiler_version: &str,
202        expected_mode: ScanDatabaseMode,
203    ) -> Result<(), String> {
204        if self.compiler_version != expected_compiler_version {
205            return Err(format!(
206                "scan database compiler version `{}` is incompatible with expected `{expected_compiler_version}`. Fix: rebuild the scan database cache with the current compiler.",
207                self.compiler_version
208            ));
209        }
210        if self.mode != expected_mode {
211            return Err(format!(
212                "scan database mode {:?} is incompatible with expected {:?}. Fix: rebuild the scan database cache for the requested scan mode.",
213                self.mode, expected_mode
214            ));
215        }
216        Ok(())
217    }
218
219    /// Validate construct-tier and dialect compatibility before trusting table
220    /// payload bytes.
221    ///
222    /// # Errors
223    ///
224    /// Returns a `Fix:` diagnostic when the construct-tier digest, dialect
225    /// digest, or reader compatibility class is not accepted by the caller.
226    pub fn validate_database_compatibility(
227        &self,
228        expected_construct_tier_digest: u64,
229        expected_dialect_digest: u64,
230        accepted_reader_compatibility: &[ScanDatabaseReaderCompatibility],
231    ) -> Result<(), String> {
232        if self.compatibility.construct_tier_digest != expected_construct_tier_digest {
233            return Err(format!(
234                "scan database construct tier digest {:#x} is incompatible with expected {expected_construct_tier_digest:#x}. Fix: rebuild the scan database cache from the current construct tier matrix.",
235                self.compatibility.construct_tier_digest
236            ));
237        }
238        if self.compatibility.dialect_digest != expected_dialect_digest {
239            return Err(format!(
240                "scan database dialect digest {:#x} is incompatible with expected {expected_dialect_digest:#x}. Fix: rebuild the scan database cache from the current regex dialect lattice.",
241                self.compatibility.dialect_digest
242            ));
243        }
244        if !accepted_reader_compatibility
245            .iter()
246            .any(|accepted| *accepted == self.compatibility.reader_compatibility)
247        {
248            return Err(format!(
249                "scan database reader compatibility {:?} is not accepted by this reader. Fix: choose a verifier-capable reader or rebuild the scan database.",
250                self.compatibility.reader_compatibility
251            ));
252        }
253        Ok(())
254    }
255}
256
257/// Encode a versioned scan database header.
258///
259/// # Errors
260///
261/// Returns [`WireEncodeErr`] when string or vector lengths exceed the bounded
262/// wire representation.
263pub fn encode_scan_database_header(
264    header: &ScanDatabaseHeader,
265) -> Result<Vec<u8>, WireEncodeErr> {
266    let mut out = Vec::with_capacity(96 + header.table_sections.len() * 25);
267    put_scan_database_header(&mut out, header)?;
268    Ok(out)
269}
270
271/// Append a versioned scan database header to an existing byte buffer.
272///
273/// # Errors
274///
275/// Returns [`WireEncodeErr`] when the header cannot be represented in the
276/// fixed-width wire format.
277pub fn put_scan_database_header(
278    out: &mut Vec<u8>,
279    header: &ScanDatabaseHeader,
280) -> Result<(), WireEncodeErr> {
281    out.extend_from_slice(SCAN_DATABASE_HEADER_MAGIC);
282    put_u32(out, SCAN_DATABASE_HEADER_VERSION);
283    out.extend_from_slice(&header.pattern_set_digest);
284    put_string(out, &header.compiler_version)?;
285    put_u8(out, header.mode.tag());
286    put_len_u32(
287        out,
288        header.table_sections.len(),
289        "scan database section count ",
290    )?;
291    for section in &header.table_sections {
292        put_u8(out, section.kind.tag());
293        put_u64(out, section.offset);
294        put_u64(out, section.byte_len);
295        put_u64(out, section.section_digest);
296    }
297    put_len_u32(
298        out,
299        header.unsupported_features.len(),
300        "scan database unsupported feature count ",
301    )?;
302    for unsupported in &header.unsupported_features {
303        put_u32(out, unsupported.pattern_index);
304        put_string(out, &unsupported.feature)?;
305    }
306    put_u64(out, header.compatibility.construct_tier_digest);
307    put_u64(out, header.compatibility.dialect_digest);
308    put_u8(out, header.compatibility.reader_compatibility.tag());
309    Ok(())
310}
311
312/// Decode a versioned scan database header without checking compiler/mode
313/// compatibility.
314///
315/// # Errors
316///
317/// Returns a `Fix:` diagnostic when the header is truncated, has the wrong
318/// magic/version, has unknown enum tags, or contains trailing bytes.
319pub fn decode_scan_database_header(bytes: &[u8]) -> Result<ScanDatabaseHeader, String> {
320    let mut reader = Reader {
321        bytes,
322        pos: 0,
323        depth: 0,
324    };
325    let magic = reader.take(SCAN_DATABASE_HEADER_MAGIC.len())?;
326    if magic != SCAN_DATABASE_HEADER_MAGIC {
327        return Err(
328            "invalid scan database header magic. Fix: load a VSDH scan database header, not a VIR0 Program blob."
329                .to_string(),
330        );
331    }
332    let version = reader.u32()?;
333    if version != SCAN_DATABASE_HEADER_VERSION {
334        return Err(format!(
335            "scan database header version {version} is unsupported; expected {SCAN_DATABASE_HEADER_VERSION}. Fix: rebuild the scan database cache."
336        ));
337    }
338    let digest_bytes = reader.take(32)?;
339    let mut pattern_set_digest = [0u8; 32];
340    pattern_set_digest.copy_from_slice(digest_bytes);
341    let compiler_version = reader.string()?;
342    let mode = ScanDatabaseMode::from_tag(reader.u8()?)?;
343
344    let section_count = reader.bounded_len(
345        MAX_SCAN_DATABASE_SECTIONS,
346        "scan database section count",
347    )?;
348    let mut table_sections = Vec::with_capacity(section_count);
349    for _ in 0..section_count {
350        table_sections.push(ScanDatabaseSectionHeader {
351            kind: ScanDatabaseSectionKind::from_tag(reader.u8()?)?,
352            offset: reader.u64()?,
353            byte_len: reader.u64()?,
354            section_digest: reader.u64()?,
355        });
356    }
357
358    let unsupported_feature_count = reader.bounded_len(
359        MAX_SCAN_DATABASE_UNSUPPORTED_FEATURES,
360        "scan database unsupported feature count",
361    )?;
362    let mut unsupported_features = Vec::with_capacity(unsupported_feature_count);
363    for _ in 0..unsupported_feature_count {
364        unsupported_features.push(UnsupportedScanFeature {
365            pattern_index: reader.u32()?,
366            feature: reader.string()?,
367        });
368    }
369
370    let compatibility = if reader.pos == bytes.len() {
371        legacy_compatibility_record(&unsupported_features)
372    } else {
373        ScanDatabaseCompatibilityRecord {
374            construct_tier_digest: reader.u64()?,
375            dialect_digest: reader.u64()?,
376            reader_compatibility: ScanDatabaseReaderCompatibility::from_tag(reader.u8()?)?,
377        }
378    };
379
380    if reader.pos != bytes.len() {
381        return Err(
382            "scan database header has trailing bytes. Fix: split the header from table payload sections before decoding."
383                .to_string(),
384        );
385    }
386
387    Ok(ScanDatabaseHeader {
388        pattern_set_digest,
389        compiler_version,
390        mode,
391        table_sections,
392        unsupported_features,
393        compatibility,
394    })
395}
396
397/// Decode and immediately validate compiler-version and mode compatibility.
398///
399/// # Errors
400///
401/// Returns the first decode or compatibility diagnostic.
402pub fn decode_compatible_scan_database_header(
403    bytes: &[u8],
404    expected_compiler_version: &str,
405    expected_mode: ScanDatabaseMode,
406) -> Result<ScanDatabaseHeader, String> {
407    let header = decode_scan_database_header(bytes)?;
408    header.validate_compatible(expected_compiler_version, expected_mode)?;
409    Ok(header)
410}
411
412/// Decode and validate compiler, mode, construct-tier, and dialect
413/// compatibility.
414///
415/// # Errors
416///
417/// Returns the first decode or compatibility diagnostic.
418pub fn decode_scan_database_header_with_compatibility(
419    bytes: &[u8],
420    expected_compiler_version: &str,
421    expected_mode: ScanDatabaseMode,
422    expected_construct_tier_digest: u64,
423    expected_dialect_digest: u64,
424    accepted_reader_compatibility: &[ScanDatabaseReaderCompatibility],
425) -> Result<ScanDatabaseHeader, String> {
426    let header =
427        decode_compatible_scan_database_header(bytes, expected_compiler_version, expected_mode)?;
428    header.validate_database_compatibility(
429        expected_construct_tier_digest,
430        expected_dialect_digest,
431        accepted_reader_compatibility,
432    )?;
433    Ok(header)
434}
435
436fn put_u64(out: &mut Vec<u8>, value: u64) {
437    out.extend_from_slice(&value.to_le_bytes());
438}
439
440fn legacy_compatibility_record(
441    unsupported_features: &[UnsupportedScanFeature],
442) -> ScanDatabaseCompatibilityRecord {
443    ScanDatabaseCompatibilityRecord {
444        construct_tier_digest: 0,
445        dialect_digest: 0,
446        reader_compatibility: if unsupported_features.is_empty() {
447            ScanDatabaseReaderCompatibility::Compatible
448        } else {
449            ScanDatabaseReaderCompatibility::RequiresVerifier
450        },
451    }
452}
453
454#[cfg(test)]
455mod tests {
456    use super::*;
457
458    const CONSTRUCT_TIER_DIGEST: u64 = 0x5ca1_c075_7e12;
459    const DIALECT_DIGEST: u64 = 0xd1a1_ec7;
460
461    fn header() -> ScanDatabaseHeader {
462        ScanDatabaseHeader {
463            pattern_set_digest: [7u8; 32],
464            compiler_version: "vyre-scan-compiler-test-v1".to_string(),
465            mode: ScanDatabaseMode::Streaming,
466            table_sections: vec![
467                ScanDatabaseSectionHeader {
468                    kind: ScanDatabaseSectionKind::LiteralTable,
469                    offset: 128,
470                    byte_len: 64,
471                    section_digest: 0x11,
472                },
473                ScanDatabaseSectionHeader {
474                    kind: ScanDatabaseSectionKind::AutomataTable,
475                    offset: 192,
476                    byte_len: 256,
477                    section_digest: 0x12,
478                },
479            ],
480            unsupported_features: vec![UnsupportedScanFeature {
481                pattern_index: 3,
482                feature: "Fix: unsupported backreference must stay verifier-only".to_string(),
483            }],
484            compatibility: ScanDatabaseCompatibilityRecord {
485                construct_tier_digest: CONSTRUCT_TIER_DIGEST,
486                dialect_digest: DIALECT_DIGEST,
487                reader_compatibility: ScanDatabaseReaderCompatibility::RequiresVerifier,
488            },
489        }
490    }
491
492    #[test]
493    fn scan_database_header_round_trips_all_fields() {
494        let original = header();
495        let bytes = encode_scan_database_header(&original).unwrap();
496        let decoded = decode_compatible_scan_database_header(
497            &bytes,
498            "vyre-scan-compiler-test-v1",
499            ScanDatabaseMode::Streaming,
500        )
501        .unwrap();
502
503        assert_eq!(decoded, original);
504        assert_eq!(decoded.section_count(), 2);
505        assert_eq!(decoded.unsupported_feature_count(), 1);
506    }
507
508    #[test]
509    fn scan_database_header_validates_construct_and_dialect_compatibility() {
510        let bytes = encode_scan_database_header(&header()).unwrap();
511        let decoded = decode_scan_database_header_with_compatibility(
512            &bytes,
513            "vyre-scan-compiler-test-v1",
514            ScanDatabaseMode::Streaming,
515            CONSTRUCT_TIER_DIGEST,
516            DIALECT_DIGEST,
517            &[ScanDatabaseReaderCompatibility::RequiresVerifier],
518        )
519        .unwrap();
520        assert_eq!(
521            decoded.compatibility.reader_compatibility,
522            ScanDatabaseReaderCompatibility::RequiresVerifier
523        );
524
525        let construct_error = decode_scan_database_header_with_compatibility(
526            &bytes,
527            "vyre-scan-compiler-test-v1",
528            ScanDatabaseMode::Streaming,
529            CONSTRUCT_TIER_DIGEST + 1,
530            DIALECT_DIGEST,
531            &[ScanDatabaseReaderCompatibility::RequiresVerifier],
532        )
533        .unwrap_err();
534        assert!(construct_error.contains("construct tier digest"));
535
536        let dialect_error = decode_scan_database_header_with_compatibility(
537            &bytes,
538            "vyre-scan-compiler-test-v1",
539            ScanDatabaseMode::Streaming,
540            CONSTRUCT_TIER_DIGEST,
541            DIALECT_DIGEST + 1,
542            &[ScanDatabaseReaderCompatibility::RequiresVerifier],
543        )
544        .unwrap_err();
545        assert!(dialect_error.contains("dialect digest"));
546    }
547
548    #[test]
549    fn scan_database_header_rejects_unaccepted_reader_compatibility() {
550        let bytes = encode_scan_database_header(&header()).unwrap();
551        let error = decode_scan_database_header_with_compatibility(
552            &bytes,
553            "vyre-scan-compiler-test-v1",
554            ScanDatabaseMode::Streaming,
555            CONSTRUCT_TIER_DIGEST,
556            DIALECT_DIGEST,
557            &[ScanDatabaseReaderCompatibility::Compatible],
558        )
559        .unwrap_err();
560
561        assert!(error.contains("reader compatibility"));
562    }
563
564    #[test]
565    fn scan_database_header_decodes_legacy_headers_with_conservative_compatibility() {
566        let mut legacy_bytes = encode_scan_database_header(&header()).unwrap();
567        legacy_bytes.truncate(legacy_bytes.len() - 17);
568        let decoded = decode_compatible_scan_database_header(
569            &legacy_bytes,
570            "vyre-scan-compiler-test-v1",
571            ScanDatabaseMode::Streaming,
572        )
573        .unwrap();
574
575        assert_eq!(decoded.compatibility.construct_tier_digest, 0);
576        assert_eq!(decoded.compatibility.dialect_digest, 0);
577        assert_eq!(
578            decoded.compatibility.reader_compatibility,
579            ScanDatabaseReaderCompatibility::RequiresVerifier
580        );
581        assert_eq!(decoded.unsupported_feature_count(), 1);
582    }
583
584    #[test]
585    fn scan_database_header_rejects_incompatible_compiler_or_mode() {
586        let bytes = encode_scan_database_header(&header()).unwrap();
587
588        let compiler_error = decode_compatible_scan_database_header(
589            &bytes,
590            "vyre-scan-compiler-test-v2",
591            ScanDatabaseMode::Streaming,
592        )
593        .unwrap_err();
594        assert!(compiler_error.contains("compiler version"));
595
596        let mode_error = decode_compatible_scan_database_header(
597            &bytes,
598            "vyre-scan-compiler-test-v1",
599            ScanDatabaseMode::Block,
600        )
601        .unwrap_err();
602        assert!(mode_error.contains("mode"));
603    }
604
605    #[test]
606    fn scan_database_header_rejects_wrong_blob_family() {
607        let error = decode_scan_database_header(b"VIR0").unwrap_err();
608        assert!(error.contains("VSDH"));
609    }
610}
vyre_foundation/serial/wire/encode/scan_database_header.rs

vyre_foundation/serial/wire/encode/
scan_database_header.rs
