vyre_foundation/serial/wire/encode/
scan_database_header.rs

1//! Versioned scan database header framing.
2//!
3//! This is not a `VIR0` Program payload. It is the canonical cache/evidence
4//! header for serialized scan databases that reference compiled pattern sets,
5//! table sections, and unsupported construct diagnostics.
6
7use super::WireEncodeErr;
8use crate::serial::wire::framing::{put_len_u32, put_string, put_u32, put_u8};
9use crate::serial::wire::Reader;
10use serde::{Deserialize, Serialize};
11
12/// Four-byte magic identifying a versioned scan database header (`VSDH`).
13pub const SCAN_DATABASE_HEADER_MAGIC: &[u8; 4] = b"VSDH";
14/// Current scan database header wire version.
15pub const SCAN_DATABASE_HEADER_VERSION: u32 = 1;
16/// Upper bound on table sections accepted from an untrusted header.
17pub const MAX_SCAN_DATABASE_SECTIONS: usize = 4_096;
18/// Upper bound on unsupported-feature diagnostics accepted from a header.
19pub const MAX_SCAN_DATABASE_UNSUPPORTED_FEATURES: usize = 4_096;
20
21/// Scan execution mode the database was compiled for.
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
23pub enum ScanDatabaseMode {
24    /// Whole-buffer block scanning.
25    Block,
26    /// Incremental streaming scan with carried state.
27    Streaming,
28    /// Vectored scan over multiple discontiguous buffers.
29    Vectored,
30}
31
32impl ScanDatabaseMode {
33    const fn tag(self) -> u8 {
34        match self {
35            Self::Block => 1,
36            Self::Streaming => 2,
37            Self::Vectored => 3,
38        }
39    }
40
41    fn from_tag(tag: u8) -> Result<Self, String> {
42        match tag {
43            1 => Ok(Self::Block),
44            2 => Ok(Self::Streaming),
45            3 => Ok(Self::Vectored),
46            _ => Err(format!(
47                "scan database mode tag {tag} is unsupported. Fix: recompile the scan database with a compatible Vyre scan compiler."
48            )),
49        }
50    }
51}
52
53/// Kind of payload a table section carries.
54#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
55pub enum ScanDatabaseSectionKind {
56    /// Compiled literal-set table.
57    LiteralTable,
58    /// Compiled automata (DFA/NFA) transition table.
59    AutomataTable,
60    /// Verifier-only regex fragments for constructs the engine cannot match.
61    VerifierFragments,
62    /// Output/result layout descriptor.
63    OutputLayout,
64    /// Carried streaming-scan state.
65    StreamingState,
66    /// Relation seed table for cross-pattern relations.
67    RelationSeeds,
68}
69
70impl ScanDatabaseSectionKind {
71    const fn tag(self) -> u8 {
72        match self {
73            Self::LiteralTable => 1,
74            Self::AutomataTable => 2,
75            Self::VerifierFragments => 3,
76            Self::OutputLayout => 4,
77            Self::StreamingState => 5,
78            Self::RelationSeeds => 6,
79        }
80    }
81
82    fn from_tag(tag: u8) -> Result<Self, String> {
83        match tag {
84            1 => Ok(Self::LiteralTable),
85            2 => Ok(Self::AutomataTable),
86            3 => Ok(Self::VerifierFragments),
87            4 => Ok(Self::OutputLayout),
88            5 => Ok(Self::StreamingState),
89            6 => Ok(Self::RelationSeeds),
90            _ => Err(format!(
91                "scan database section tag {tag} is unsupported. Fix: recompile the scan database with a compatible Vyre scan compiler."
92            )),
93        }
94    }
95}
96
97/// Locator and integrity digest for one table section in the database body.
98#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
99pub struct ScanDatabaseSectionHeader {
100    /// Kind of payload this section carries.
101    pub kind: ScanDatabaseSectionKind,
102    /// Byte offset of the section payload from the start of the database body.
103    pub offset: u64,
104    /// Byte length of the section payload.
105    pub byte_len: u64,
106    /// Integrity digest over the section payload bytes.
107    pub section_digest: u64,
108}
109
110/// A construct the compiler could not lower, recorded for verifier handoff.
111#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
112pub struct UnsupportedScanFeature {
113    /// Index of the pattern that used the unsupported construct.
114    pub pattern_index: u32,
115    /// Human-readable description of the unsupported feature.
116    pub feature: String,
117}
118
119/// How a reader may consume a database given its unsupported constructs.
120#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
121pub enum ScanDatabaseReaderCompatibility {
122    /// Fully consumable by the engine alone.
123    Compatible,
124    /// Consumable only with a verifier for the unsupported fragments.
125    RequiresVerifier,
126    /// Not consumable by this reader at all.
127    Incompatible,
128}
129
130impl ScanDatabaseReaderCompatibility {
131    const fn tag(self) -> u8 {
132        match self {
133            Self::Compatible => 1,
134            Self::RequiresVerifier => 2,
135            Self::Incompatible => 3,
136        }
137    }
138
139    fn from_tag(tag: u8) -> Result<Self, String> {
140        match tag {
141            1 => Ok(Self::Compatible),
142            2 => Ok(Self::RequiresVerifier),
143            3 => Ok(Self::Incompatible),
144            _ => Err(format!(
145                "scan database reader compatibility tag {tag} is unsupported. Fix: rebuild the scan database cache with a compatible Vyre scan compiler."
146            )),
147        }
148    }
149}
150
151/// Construct-tier and dialect compatibility fingerprint for a database.
152#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
153pub struct ScanDatabaseCompatibilityRecord {
154    /// Digest of the construct-tier matrix the database was compiled against.
155    pub construct_tier_digest: u64,
156    /// Digest of the regex dialect lattice the database was compiled against.
157    pub dialect_digest: u64,
158    /// Reader compatibility class for this database.
159    pub reader_compatibility: ScanDatabaseReaderCompatibility,
160}
161
162/// Versioned, self-describing header for a serialized scan database.
163#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
164pub struct ScanDatabaseHeader {
165    /// 32-byte digest of the compiled pattern set.
166    pub pattern_set_digest: [u8; 32],
167    /// Version string of the compiler that produced the database.
168    pub compiler_version: String,
169    /// Scan mode the database was compiled for.
170    pub mode: ScanDatabaseMode,
171    /// Locators for the table sections in the database body.
172    pub table_sections: Vec<ScanDatabaseSectionHeader>,
173    /// Constructs that require verifier handoff.
174    pub unsupported_features: Vec<UnsupportedScanFeature>,
175    /// Construct-tier and dialect compatibility fingerprint.
176    pub compatibility: ScanDatabaseCompatibilityRecord,
177}
178
179impl ScanDatabaseHeader {
180    /// Number of table sections in the database body.
181    #[must_use]
182    pub fn section_count(&self) -> usize {
183        self.table_sections.len()
184    }
185
186    /// Number of unsupported-feature diagnostics recorded in the header.
187    #[must_use]
188    pub fn unsupported_feature_count(&self) -> usize {
189        self.unsupported_features.len()
190    }
191
192    /// Validate this header against the consumer's required compiler version
193    /// and scan mode before any table payload is trusted.
194    ///
195    /// # Errors
196    ///
197    /// Returns an actionable `Fix:` diagnostic when compiler version or mode
198    /// differs from the expected cache key.
199    pub fn validate_compatible(
200        &self,
201        expected_compiler_version: &str,
202        expected_mode: ScanDatabaseMode,
203    ) -> Result<(), String> {
204        if self.compiler_version != expected_compiler_version {
205            return Err(format!(
206                "scan database compiler version `{}` is incompatible with expected `{expected_compiler_version}`. Fix: rebuild the scan database cache with the current compiler.",
207                self.compiler_version
208            ));
209        }
210        if self.mode != expected_mode {
211            return Err(format!(
212                "scan database mode {:?} is incompatible with expected {:?}. Fix: rebuild the scan database cache for the requested scan mode.",
213                self.mode, expected_mode
214            ));
215        }
216        Ok(())
217    }
218
219    /// Validate construct-tier and dialect compatibility before trusting table
220    /// payload bytes.
221    ///
222    /// # Errors
223    ///
224    /// Returns a `Fix:` diagnostic when the construct-tier digest, dialect
225    /// digest, or reader compatibility class is not accepted by the caller.
226    pub fn validate_database_compatibility(
227        &self,
228        expected_construct_tier_digest: u64,
229        expected_dialect_digest: u64,
230        accepted_reader_compatibility: &[ScanDatabaseReaderCompatibility],
231    ) -> Result<(), String> {
232        if self.compatibility.construct_tier_digest != expected_construct_tier_digest {
233            return Err(format!(
234                "scan database construct tier digest {:#x} is incompatible with expected {expected_construct_tier_digest:#x}. Fix: rebuild the scan database cache from the current construct tier matrix.",
235                self.compatibility.construct_tier_digest
236            ));
237        }
238        if self.compatibility.dialect_digest != expected_dialect_digest {
239            return Err(format!(
240                "scan database dialect digest {:#x} is incompatible with expected {expected_dialect_digest:#x}. Fix: rebuild the scan database cache from the current regex dialect lattice.",
241                self.compatibility.dialect_digest
242            ));
243        }
244        if !accepted_reader_compatibility
245            .iter()
246            .any(|accepted| *accepted == self.compatibility.reader_compatibility)
247        {
248            return Err(format!(
249                "scan database reader compatibility {:?} is not accepted by this reader. Fix: choose a verifier-capable reader or rebuild the scan database.",
250                self.compatibility.reader_compatibility
251            ));
252        }
253        Ok(())
254    }
255}
256
257/// Encode a versioned scan database header.
258///
259/// # Errors
260///
261/// Returns [`WireEncodeErr`] when string or vector lengths exceed the bounded
262/// wire representation.
263pub fn encode_scan_database_header(header: &ScanDatabaseHeader) -> Result<Vec<u8>, WireEncodeErr> {
264    let mut out = Vec::with_capacity(96 + header.table_sections.len() * 25);
265    put_scan_database_header(&mut out, header)?;
266    Ok(out)
267}
268
269/// Append a versioned scan database header to an existing byte buffer.
270///
271/// # Errors
272///
273/// Returns [`WireEncodeErr`] when the header cannot be represented in the
274/// fixed-width wire format.
275pub fn put_scan_database_header(
276    out: &mut Vec<u8>,
277    header: &ScanDatabaseHeader,
278) -> Result<(), WireEncodeErr> {
279    out.extend_from_slice(SCAN_DATABASE_HEADER_MAGIC);
280    put_u32(out, SCAN_DATABASE_HEADER_VERSION);
281    out.extend_from_slice(&header.pattern_set_digest);
282    put_string(out, &header.compiler_version)?;
283    put_u8(out, header.mode.tag());
284    put_len_u32(
285        out,
286        header.table_sections.len(),
287        "scan database section count ",
288    )?;
289    for section in &header.table_sections {
290        put_u8(out, section.kind.tag());
291        put_u64(out, section.offset);
292        put_u64(out, section.byte_len);
293        put_u64(out, section.section_digest);
294    }
295    put_len_u32(
296        out,
297        header.unsupported_features.len(),
298        "scan database unsupported feature count ",
299    )?;
300    for unsupported in &header.unsupported_features {
301        put_u32(out, unsupported.pattern_index);
302        put_string(out, &unsupported.feature)?;
303    }
304    put_u64(out, header.compatibility.construct_tier_digest);
305    put_u64(out, header.compatibility.dialect_digest);
306    put_u8(out, header.compatibility.reader_compatibility.tag());
307    Ok(())
308}
309
310/// Decode a versioned scan database header without checking compiler/mode
311/// compatibility.
312///
313/// # Errors
314///
315/// Returns a `Fix:` diagnostic when the header is truncated, has the wrong
316/// magic/version, has unknown enum tags, or contains trailing bytes.
317pub fn decode_scan_database_header(bytes: &[u8]) -> Result<ScanDatabaseHeader, String> {
318    let mut reader = Reader {
319        bytes,
320        pos: 0,
321        depth: 0,
322    };
323    let magic = reader.take(SCAN_DATABASE_HEADER_MAGIC.len())?;
324    if magic != SCAN_DATABASE_HEADER_MAGIC {
325        return Err(
326            "invalid scan database header magic. Fix: load a VSDH scan database header, not a VIR0 Program blob."
327                .to_string(),
328        );
329    }
330    let version = reader.u32()?;
331    if version != SCAN_DATABASE_HEADER_VERSION {
332        return Err(format!(
333            "scan database header version {version} is unsupported; expected {SCAN_DATABASE_HEADER_VERSION}. Fix: rebuild the scan database cache."
334        ));
335    }
336    let digest_bytes = reader.take(32)?;
337    let mut pattern_set_digest = [0u8; 32];
338    pattern_set_digest.copy_from_slice(digest_bytes);
339    let compiler_version = reader.string()?;
340    let mode = ScanDatabaseMode::from_tag(reader.u8()?)?;
341
342    let section_count =
343        reader.bounded_len(MAX_SCAN_DATABASE_SECTIONS, "scan database section count")?;
344    let mut table_sections = Vec::with_capacity(section_count);
345    for _ in 0..section_count {
346        table_sections.push(ScanDatabaseSectionHeader {
347            kind: ScanDatabaseSectionKind::from_tag(reader.u8()?)?,
348            offset: reader.u64()?,
349            byte_len: reader.u64()?,
350            section_digest: reader.u64()?,
351        });
352    }
353
354    let unsupported_feature_count = reader.bounded_len(
355        MAX_SCAN_DATABASE_UNSUPPORTED_FEATURES,
356        "scan database unsupported feature count",
357    )?;
358    let mut unsupported_features = Vec::with_capacity(unsupported_feature_count);
359    for _ in 0..unsupported_feature_count {
360        unsupported_features.push(UnsupportedScanFeature {
361            pattern_index: reader.u32()?,
362            feature: reader.string()?,
363        });
364    }
365
366    let compatibility = if reader.pos == bytes.len() {
367        legacy_compatibility_record(&unsupported_features)
368    } else {
369        ScanDatabaseCompatibilityRecord {
370            construct_tier_digest: reader.u64()?,
371            dialect_digest: reader.u64()?,
372            reader_compatibility: ScanDatabaseReaderCompatibility::from_tag(reader.u8()?)?,
373        }
374    };
375
376    if reader.pos != bytes.len() {
377        return Err(
378            "scan database header has trailing bytes. Fix: split the header from table payload sections before decoding."
379                .to_string(),
380        );
381    }
382
383    Ok(ScanDatabaseHeader {
384        pattern_set_digest,
385        compiler_version,
386        mode,
387        table_sections,
388        unsupported_features,
389        compatibility,
390    })
391}
392
393/// Decode and immediately validate compiler-version and mode compatibility.
394///
395/// # Errors
396///
397/// Returns the first decode or compatibility diagnostic.
398pub fn decode_compatible_scan_database_header(
399    bytes: &[u8],
400    expected_compiler_version: &str,
401    expected_mode: ScanDatabaseMode,
402) -> Result<ScanDatabaseHeader, String> {
403    let header = decode_scan_database_header(bytes)?;
404    header.validate_compatible(expected_compiler_version, expected_mode)?;
405    Ok(header)
406}
407
408/// Decode and validate compiler, mode, construct-tier, and dialect
409/// compatibility.
410///
411/// # Errors
412///
413/// Returns the first decode or compatibility diagnostic.
414pub fn decode_scan_database_header_with_compatibility(
415    bytes: &[u8],
416    expected_compiler_version: &str,
417    expected_mode: ScanDatabaseMode,
418    expected_construct_tier_digest: u64,
419    expected_dialect_digest: u64,
420    accepted_reader_compatibility: &[ScanDatabaseReaderCompatibility],
421) -> Result<ScanDatabaseHeader, String> {
422    let header =
423        decode_compatible_scan_database_header(bytes, expected_compiler_version, expected_mode)?;
424    header.validate_database_compatibility(
425        expected_construct_tier_digest,
426        expected_dialect_digest,
427        accepted_reader_compatibility,
428    )?;
429    Ok(header)
430}
431
432fn put_u64(out: &mut Vec<u8>, value: u64) {
433    out.extend_from_slice(&value.to_le_bytes());
434}
435
436fn legacy_compatibility_record(
437    unsupported_features: &[UnsupportedScanFeature],
438) -> ScanDatabaseCompatibilityRecord {
439    ScanDatabaseCompatibilityRecord {
440        construct_tier_digest: 0,
441        dialect_digest: 0,
442        reader_compatibility: if unsupported_features.is_empty() {
443            ScanDatabaseReaderCompatibility::Compatible
444        } else {
445            ScanDatabaseReaderCompatibility::RequiresVerifier
446        },
447    }
448}
449
450#[cfg(test)]
451mod tests {
452    use super::*;
453
454    const CONSTRUCT_TIER_DIGEST: u64 = 0x5ca1_c075_7e12;
455    const DIALECT_DIGEST: u64 = 0xd1a1_ec7;
456
457    fn header() -> ScanDatabaseHeader {
458        ScanDatabaseHeader {
459            pattern_set_digest: [7u8; 32],
460            compiler_version: "vyre-scan-compiler-test-v1".to_string(),
461            mode: ScanDatabaseMode::Streaming,
462            table_sections: vec![
463                ScanDatabaseSectionHeader {
464                    kind: ScanDatabaseSectionKind::LiteralTable,
465                    offset: 128,
466                    byte_len: 64,
467                    section_digest: 0x11,
468                },
469                ScanDatabaseSectionHeader {
470                    kind: ScanDatabaseSectionKind::AutomataTable,
471                    offset: 192,
472                    byte_len: 256,
473                    section_digest: 0x12,
474                },
475            ],
476            unsupported_features: vec![UnsupportedScanFeature {
477                pattern_index: 3,
478                feature: "Fix: unsupported backreference must stay verifier-only".to_string(),
479            }],
480            compatibility: ScanDatabaseCompatibilityRecord {
481                construct_tier_digest: CONSTRUCT_TIER_DIGEST,
482                dialect_digest: DIALECT_DIGEST,
483                reader_compatibility: ScanDatabaseReaderCompatibility::RequiresVerifier,
484            },
485        }
486    }
487
488    #[test]
489    fn scan_database_header_round_trips_all_fields() {
490        let original = header();
491        let bytes = encode_scan_database_header(&original).unwrap();
492        let decoded = decode_compatible_scan_database_header(
493            &bytes,
494            "vyre-scan-compiler-test-v1",
495            ScanDatabaseMode::Streaming,
496        )
497        .unwrap();
498
499        assert_eq!(decoded, original);
500        assert_eq!(decoded.section_count(), 2);
501        assert_eq!(decoded.unsupported_feature_count(), 1);
502    }
503
504    #[test]
505    fn scan_database_header_validates_construct_and_dialect_compatibility() {
506        let bytes = encode_scan_database_header(&header()).unwrap();
507        let decoded = decode_scan_database_header_with_compatibility(
508            &bytes,
509            "vyre-scan-compiler-test-v1",
510            ScanDatabaseMode::Streaming,
511            CONSTRUCT_TIER_DIGEST,
512            DIALECT_DIGEST,
513            &[ScanDatabaseReaderCompatibility::RequiresVerifier],
514        )
515        .unwrap();
516        assert_eq!(
517            decoded.compatibility.reader_compatibility,
518            ScanDatabaseReaderCompatibility::RequiresVerifier
519        );
520
521        let construct_error = decode_scan_database_header_with_compatibility(
522            &bytes,
523            "vyre-scan-compiler-test-v1",
524            ScanDatabaseMode::Streaming,
525            CONSTRUCT_TIER_DIGEST + 1,
526            DIALECT_DIGEST,
527            &[ScanDatabaseReaderCompatibility::RequiresVerifier],
528        )
529        .unwrap_err();
530        assert!(construct_error.contains("construct tier digest"));
531
532        let dialect_error = decode_scan_database_header_with_compatibility(
533            &bytes,
534            "vyre-scan-compiler-test-v1",
535            ScanDatabaseMode::Streaming,
536            CONSTRUCT_TIER_DIGEST,
537            DIALECT_DIGEST + 1,
538            &[ScanDatabaseReaderCompatibility::RequiresVerifier],
539        )
540        .unwrap_err();
541        assert!(dialect_error.contains("dialect digest"));
542    }
543
544    #[test]
545    fn scan_database_header_rejects_unaccepted_reader_compatibility() {
546        let bytes = encode_scan_database_header(&header()).unwrap();
547        let error = decode_scan_database_header_with_compatibility(
548            &bytes,
549            "vyre-scan-compiler-test-v1",
550            ScanDatabaseMode::Streaming,
551            CONSTRUCT_TIER_DIGEST,
552            DIALECT_DIGEST,
553            &[ScanDatabaseReaderCompatibility::Compatible],
554        )
555        .unwrap_err();
556
557        assert!(error.contains("reader compatibility"));
558    }
559
560    #[test]
561    fn scan_database_header_decodes_legacy_headers_with_conservative_compatibility() {
562        let mut legacy_bytes = encode_scan_database_header(&header()).unwrap();
563        legacy_bytes.truncate(legacy_bytes.len() - 17);
564        let decoded = decode_compatible_scan_database_header(
565            &legacy_bytes,
566            "vyre-scan-compiler-test-v1",
567            ScanDatabaseMode::Streaming,
568        )
569        .unwrap();
570
571        assert_eq!(decoded.compatibility.construct_tier_digest, 0);
572        assert_eq!(decoded.compatibility.dialect_digest, 0);
573        assert_eq!(
574            decoded.compatibility.reader_compatibility,
575            ScanDatabaseReaderCompatibility::RequiresVerifier
576        );
577        assert_eq!(decoded.unsupported_feature_count(), 1);
578    }
579
580    #[test]
581    fn scan_database_header_rejects_incompatible_compiler_or_mode() {
582        let bytes = encode_scan_database_header(&header()).unwrap();
583
584        let compiler_error = decode_compatible_scan_database_header(
585            &bytes,
586            "vyre-scan-compiler-test-v2",
587            ScanDatabaseMode::Streaming,
588        )
589        .unwrap_err();
590        assert!(compiler_error.contains("compiler version"));
591
592        let mode_error = decode_compatible_scan_database_header(
593            &bytes,
594            "vyre-scan-compiler-test-v1",
595            ScanDatabaseMode::Block,
596        )
597        .unwrap_err();
598        assert!(mode_error.contains("mode"));
599    }
600
601    #[test]
602    fn scan_database_header_rejects_wrong_blob_family() {
603        let error = decode_scan_database_header(b"VIR0").unwrap_err();
604        assert!(error.contains("VSDH"));
605    }
606}
vyre_foundation/serial/wire/encode/scan_database_header.rs

vyre_foundation/serial/wire/encode/
scan_database_header.rs
