reddb_file/
table_def_codec.rs

1//! On-disk codec for the `RTBL` table-definition payload.
2//!
3//! `reddb-file` already owns the opaque `table_def_hex` field of the physical
4//! collection contract; this module makes it the authority for the inner byte
5//! layout too (ADR 0046). The codec operates on a plain [`TableDefLayout`] whose
6//! type/index/constraint discriminants are raw bytes — the server engine owns
7//! the SQL-level `DataType`/`IndexType`/`ConstraintType` enums and maps them to
8//! and from these bytes. The magic, version, LEB128 varint + length-prefixed
9//! string framing, and field ordering all live here.
10
11/// Magic prefixing a persisted table definition.
12pub const TABLE_DEF_MAGIC: &[u8; 4] = b"RTBL";
13
14/// Plain, engine-agnostic view of a persisted column definition.
15#[derive(Debug, Clone, PartialEq)]
16pub struct ColumnLayout {
17    /// Column name.
18    pub name: String,
19    /// `DataType` discriminant byte.
20    pub data_type: u8,
21    /// Whether NULL is allowed.
22    pub nullable: bool,
23    /// Serialized default value, if any.
24    pub default: Option<Vec<u8>>,
25    /// Vector dimension, if any.
26    pub vector_dim: Option<u32>,
27    /// Per-column compression flag.
28    pub compress: bool,
29    /// Enum variant labels (for enum-typed columns).
30    pub enum_variants: Vec<String>,
31    /// Decimal precision.
32    pub decimal_precision: u8,
33    /// Array element `DataType` discriminant byte, if any.
34    pub element_type: Option<u8>,
35    /// Column metadata `(key, value)` pairs in persistence order.
36    pub metadata: Vec<(String, String)>,
37}
38
39/// Plain, engine-agnostic view of a persisted index definition.
40#[derive(Debug, Clone, PartialEq)]
41pub struct IndexLayout {
42    /// Index name.
43    pub name: String,
44    /// `IndexType` discriminant byte.
45    pub index_type: u8,
46    /// Whether the index is unique.
47    pub unique: bool,
48    /// Indexed column names in order.
49    pub columns: Vec<String>,
50}
51
52/// Plain, engine-agnostic view of a persisted constraint definition.
53#[derive(Debug, Clone, PartialEq)]
54pub struct ConstraintLayout {
55    /// Constraint name.
56    pub name: String,
57    /// `ConstraintType` discriminant byte.
58    pub constraint_type: u8,
59    /// Constrained column names.
60    pub columns: Vec<String>,
61    /// Referenced table (foreign keys).
62    pub ref_table: Option<String>,
63    /// Referenced columns (foreign keys).
64    pub ref_columns: Option<Vec<String>>,
65}
66
67/// Plain, engine-agnostic view of a persisted table definition.
68#[derive(Debug, Clone, PartialEq)]
69pub struct TableDefLayout {
70    /// Schema version field.
71    pub version: u32,
72    /// Table name.
73    pub name: String,
74    /// Creation timestamp.
75    pub created_at: u64,
76    /// Last-update timestamp.
77    pub updated_at: u64,
78    /// Columns in declaration order.
79    pub columns: Vec<ColumnLayout>,
80    /// Primary-key column names.
81    pub primary_key: Vec<String>,
82    /// Indexes.
83    pub indexes: Vec<IndexLayout>,
84    /// Constraints.
85    pub constraints: Vec<ConstraintLayout>,
86}
87
88/// Errors raised while decoding a persisted table definition.
89#[derive(Debug, Clone, PartialEq, Eq)]
90pub enum TableDefCodecError {
91    /// The payload was shorter than required.
92    TruncatedData,
93    /// The leading magic was not `RTBL`.
94    InvalidMagic,
95    /// A varint exceeded 64 bits.
96    VarintOverflow,
97    /// A length-prefixed string contained invalid UTF-8.
98    InvalidUtf8,
99}
100
101impl std::fmt::Display for TableDefCodecError {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        match self {
104            TableDefCodecError::TruncatedData => write!(f, "truncated data"),
105            TableDefCodecError::InvalidMagic => write!(f, "invalid magic bytes"),
106            TableDefCodecError::VarintOverflow => write!(f, "varint overflow"),
107            TableDefCodecError::InvalidUtf8 => write!(f, "invalid utf-8"),
108        }
109    }
110}
111
112impl std::error::Error for TableDefCodecError {}
113
114// ---------------------------------------------------------------------------
115// Encode
116// ---------------------------------------------------------------------------
117
118/// Encode a table definition (`RTBL`), byte-faithful to the legacy server.
119pub fn encode_table_def(layout: &TableDefLayout) -> Vec<u8> {
120    let mut buf = Vec::new();
121
122    buf.extend_from_slice(TABLE_DEF_MAGIC);
123    buf.extend_from_slice(&layout.version.to_le_bytes());
124    write_string(&mut buf, &layout.name);
125    buf.extend_from_slice(&layout.created_at.to_le_bytes());
126    buf.extend_from_slice(&layout.updated_at.to_le_bytes());
127
128    write_varint(&mut buf, layout.columns.len() as u64);
129    for col in &layout.columns {
130        write_column(&mut buf, col);
131    }
132
133    write_varint(&mut buf, layout.primary_key.len() as u64);
134    for pk in &layout.primary_key {
135        write_string(&mut buf, pk);
136    }
137
138    write_varint(&mut buf, layout.indexes.len() as u64);
139    for idx in &layout.indexes {
140        write_index(&mut buf, idx);
141    }
142
143    write_varint(&mut buf, layout.constraints.len() as u64);
144    for constraint in &layout.constraints {
145        write_constraint(&mut buf, constraint);
146    }
147
148    buf
149}
150
151fn write_column(buf: &mut Vec<u8>, col: &ColumnLayout) {
152    write_string(buf, &col.name);
153    buf.push(col.data_type);
154    buf.push(if col.nullable { 1 } else { 0 });
155
156    if let Some(ref default) = col.default {
157        buf.push(1);
158        write_varint(buf, default.len() as u64);
159        buf.extend_from_slice(default);
160    } else {
161        buf.push(0);
162    }
163
164    if let Some(dim) = col.vector_dim {
165        buf.push(1);
166        buf.extend_from_slice(&dim.to_le_bytes());
167    } else {
168        buf.push(0);
169    }
170
171    buf.push(if col.compress { 1 } else { 0 });
172
173    write_varint(buf, col.enum_variants.len() as u64);
174    for variant in &col.enum_variants {
175        write_string(buf, variant);
176    }
177
178    buf.push(col.decimal_precision);
179
180    if let Some(et) = col.element_type {
181        buf.push(1);
182        buf.push(et);
183    } else {
184        buf.push(0);
185    }
186
187    write_varint(buf, col.metadata.len() as u64);
188    for (k, v) in &col.metadata {
189        write_string(buf, k);
190        write_string(buf, v);
191    }
192}
193
194fn write_index(buf: &mut Vec<u8>, idx: &IndexLayout) {
195    write_string(buf, &idx.name);
196    buf.push(idx.index_type);
197    buf.push(if idx.unique { 1 } else { 0 });
198    write_varint(buf, idx.columns.len() as u64);
199    for col in &idx.columns {
200        write_string(buf, col);
201    }
202}
203
204fn write_constraint(buf: &mut Vec<u8>, constraint: &ConstraintLayout) {
205    write_string(buf, &constraint.name);
206    buf.push(constraint.constraint_type);
207
208    write_varint(buf, constraint.columns.len() as u64);
209    for col in &constraint.columns {
210        write_string(buf, col);
211    }
212
213    if let Some(ref table) = constraint.ref_table {
214        buf.push(1);
215        write_string(buf, table);
216        if let Some(ref cols) = constraint.ref_columns {
217            write_varint(buf, cols.len() as u64);
218            for col in cols {
219                write_string(buf, col);
220            }
221        } else {
222            write_varint(buf, 0);
223        }
224    } else {
225        buf.push(0);
226    }
227}
228
229// ---------------------------------------------------------------------------
230// Decode
231// ---------------------------------------------------------------------------
232
233/// Decode a table definition (`RTBL`) produced by [`encode_table_def`] or the
234/// legacy server `to_bytes`.
235pub fn decode_table_def(data: &[u8]) -> Result<TableDefLayout, TableDefCodecError> {
236    if data.len() < 4 {
237        return Err(TableDefCodecError::TruncatedData);
238    }
239    if &data[0..4] != TABLE_DEF_MAGIC {
240        return Err(TableDefCodecError::InvalidMagic);
241    }
242
243    let mut offset = 4;
244
245    if data.len() < offset + 4 {
246        return Err(TableDefCodecError::TruncatedData);
247    }
248    let version = u32::from_le_bytes(data[offset..offset + 4].try_into().unwrap());
249    offset += 4;
250
251    let (name, name_len) = read_string(&data[offset..])?;
252    offset += name_len;
253
254    if data.len() < offset + 16 {
255        return Err(TableDefCodecError::TruncatedData);
256    }
257    let created_at = u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap());
258    offset += 8;
259    let updated_at = u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap());
260    offset += 8;
261
262    let (col_count, varint_len) = read_varint(&data[offset..])?;
263    offset += varint_len;
264    let mut columns = Vec::with_capacity(col_count as usize);
265    for _ in 0..col_count {
266        let (col, col_len) = read_column(&data[offset..])?;
267        offset += col_len;
268        columns.push(col);
269    }
270
271    let (pk_count, varint_len) = read_varint(&data[offset..])?;
272    offset += varint_len;
273    let mut primary_key = Vec::with_capacity(pk_count as usize);
274    for _ in 0..pk_count {
275        let (pk, pk_len) = read_string(&data[offset..])?;
276        offset += pk_len;
277        primary_key.push(pk);
278    }
279
280    let (idx_count, varint_len) = read_varint(&data[offset..])?;
281    offset += varint_len;
282    let mut indexes = Vec::with_capacity(idx_count as usize);
283    for _ in 0..idx_count {
284        let (idx, idx_len) = read_index(&data[offset..])?;
285        offset += idx_len;
286        indexes.push(idx);
287    }
288
289    let (constraint_count, varint_len) = read_varint(&data[offset..])?;
290    offset += varint_len;
291    let mut constraints = Vec::with_capacity(constraint_count as usize);
292    for _ in 0..constraint_count {
293        let (constraint, constraint_len) = read_constraint(&data[offset..])?;
294        offset += constraint_len;
295        constraints.push(constraint);
296    }
297
298    Ok(TableDefLayout {
299        version,
300        name,
301        created_at,
302        updated_at,
303        columns,
304        primary_key,
305        indexes,
306        constraints,
307    })
308}
309
310fn read_column(data: &[u8]) -> Result<(ColumnLayout, usize), TableDefCodecError> {
311    let mut offset = 0;
312
313    let (name, name_len) = read_string(&data[offset..])?;
314    offset += name_len;
315
316    if data.len() < offset + 2 {
317        return Err(TableDefCodecError::TruncatedData);
318    }
319    let data_type = data[offset];
320    offset += 1;
321    let nullable = data[offset] != 0;
322    offset += 1;
323
324    if data.len() < offset + 1 {
325        return Err(TableDefCodecError::TruncatedData);
326    }
327    let has_default = data[offset] != 0;
328    offset += 1;
329    let default = if has_default {
330        let (len, varint_len) = read_varint(&data[offset..])?;
331        offset += varint_len;
332        if data.len() < offset + len as usize {
333            return Err(TableDefCodecError::TruncatedData);
334        }
335        let default_data = data[offset..offset + len as usize].to_vec();
336        offset += len as usize;
337        Some(default_data)
338    } else {
339        None
340    };
341
342    if data.len() < offset + 1 {
343        return Err(TableDefCodecError::TruncatedData);
344    }
345    let has_vector_dim = data[offset] != 0;
346    offset += 1;
347    let vector_dim = if has_vector_dim {
348        if data.len() < offset + 4 {
349            return Err(TableDefCodecError::TruncatedData);
350        }
351        let dim = u32::from_le_bytes(data[offset..offset + 4].try_into().unwrap());
352        offset += 4;
353        Some(dim)
354    } else {
355        None
356    };
357
358    if data.len() < offset + 1 {
359        return Err(TableDefCodecError::TruncatedData);
360    }
361    let compress = data[offset] != 0;
362    offset += 1;
363
364    let (variant_count, varint_len) = read_varint(&data[offset..])?;
365    offset += varint_len;
366    let mut enum_variants = Vec::with_capacity(variant_count as usize);
367    for _ in 0..variant_count {
368        let (variant, variant_len) = read_string(&data[offset..])?;
369        offset += variant_len;
370        enum_variants.push(variant);
371    }
372
373    if data.len() < offset + 1 {
374        return Err(TableDefCodecError::TruncatedData);
375    }
376    let decimal_precision = data[offset];
377    offset += 1;
378
379    if data.len() < offset + 1 {
380        return Err(TableDefCodecError::TruncatedData);
381    }
382    let has_element_type = data[offset] != 0;
383    offset += 1;
384    let element_type = if has_element_type {
385        if data.len() < offset + 1 {
386            return Err(TableDefCodecError::TruncatedData);
387        }
388        let et = data[offset];
389        offset += 1;
390        Some(et)
391    } else {
392        None
393    };
394
395    let (meta_count, varint_len) = read_varint(&data[offset..])?;
396    offset += varint_len;
397    let mut metadata = Vec::with_capacity(meta_count as usize);
398    for _ in 0..meta_count {
399        let (k, k_len) = read_string(&data[offset..])?;
400        offset += k_len;
401        let (v, v_len) = read_string(&data[offset..])?;
402        offset += v_len;
403        metadata.push((k, v));
404    }
405
406    Ok((
407        ColumnLayout {
408            name,
409            data_type,
410            nullable,
411            default,
412            vector_dim,
413            compress,
414            enum_variants,
415            decimal_precision,
416            element_type,
417            metadata,
418        },
419        offset,
420    ))
421}
422
423fn read_index(data: &[u8]) -> Result<(IndexLayout, usize), TableDefCodecError> {
424    let mut offset = 0;
425
426    let (name, name_len) = read_string(&data[offset..])?;
427    offset += name_len;
428
429    if data.len() < offset + 2 {
430        return Err(TableDefCodecError::TruncatedData);
431    }
432    let index_type = data[offset];
433    offset += 1;
434    let unique = data[offset] != 0;
435    offset += 1;
436
437    let (col_count, varint_len) = read_varint(&data[offset..])?;
438    offset += varint_len;
439    let mut columns = Vec::with_capacity(col_count as usize);
440    for _ in 0..col_count {
441        let (col, col_len) = read_string(&data[offset..])?;
442        offset += col_len;
443        columns.push(col);
444    }
445
446    Ok((
447        IndexLayout {
448            name,
449            index_type,
450            unique,
451            columns,
452        },
453        offset,
454    ))
455}
456
457fn read_constraint(data: &[u8]) -> Result<(ConstraintLayout, usize), TableDefCodecError> {
458    let mut offset = 0;
459
460    let (name, name_len) = read_string(&data[offset..])?;
461    offset += name_len;
462
463    if data.len() < offset + 1 {
464        return Err(TableDefCodecError::TruncatedData);
465    }
466    let constraint_type = data[offset];
467    offset += 1;
468
469    let (col_count, varint_len) = read_varint(&data[offset..])?;
470    offset += varint_len;
471    let mut columns = Vec::with_capacity(col_count as usize);
472    for _ in 0..col_count {
473        let (col, col_len) = read_string(&data[offset..])?;
474        offset += col_len;
475        columns.push(col);
476    }
477
478    if data.len() < offset + 1 {
479        return Err(TableDefCodecError::TruncatedData);
480    }
481    let has_ref = data[offset] != 0;
482    offset += 1;
483
484    let (ref_table, ref_columns) = if has_ref {
485        let (table, table_len) = read_string(&data[offset..])?;
486        offset += table_len;
487
488        let (ref_col_count, varint_len) = read_varint(&data[offset..])?;
489        offset += varint_len;
490
491        let mut ref_cols = Vec::with_capacity(ref_col_count as usize);
492        for _ in 0..ref_col_count {
493            let (col, col_len) = read_string(&data[offset..])?;
494            offset += col_len;
495            ref_cols.push(col);
496        }
497
498        (Some(table), Some(ref_cols))
499    } else {
500        (None, None)
501    };
502
503    Ok((
504        ConstraintLayout {
505            name,
506            constraint_type,
507            columns,
508            ref_table,
509            ref_columns,
510        },
511        offset,
512    ))
513}
514
515// ---------------------------------------------------------------------------
516// LEB128 varint + length-prefixed string framing
517// ---------------------------------------------------------------------------
518
519fn write_varint(buf: &mut Vec<u8>, mut value: u64) {
520    loop {
521        let mut byte = (value & 0x7F) as u8;
522        value >>= 7;
523        if value != 0 {
524            byte |= 0x80;
525        }
526        buf.push(byte);
527        if value == 0 {
528            break;
529        }
530    }
531}
532
533fn read_varint(data: &[u8]) -> Result<(u64, usize), TableDefCodecError> {
534    let mut result: u64 = 0;
535    let mut shift = 0;
536    let mut offset = 0;
537
538    loop {
539        if offset >= data.len() {
540            return Err(TableDefCodecError::TruncatedData);
541        }
542        let byte = data[offset];
543        offset += 1;
544
545        if shift >= 64 {
546            return Err(TableDefCodecError::VarintOverflow);
547        }
548
549        result |= ((byte & 0x7F) as u64) << shift;
550        shift += 7;
551
552        if byte & 0x80 == 0 {
553            break;
554        }
555    }
556
557    Ok((result, offset))
558}
559
560fn write_string(buf: &mut Vec<u8>, s: &str) {
561    let bytes = s.as_bytes();
562    write_varint(buf, bytes.len() as u64);
563    buf.extend_from_slice(bytes);
564}
565
566fn read_string(data: &[u8]) -> Result<(String, usize), TableDefCodecError> {
567    let (len, varint_len) = read_varint(data)?;
568    let offset = varint_len;
569    if data.len() < offset + len as usize {
570        return Err(TableDefCodecError::TruncatedData);
571    }
572    let s = String::from_utf8(data[offset..offset + len as usize].to_vec())
573        .map_err(|_| TableDefCodecError::InvalidUtf8)?;
574    Ok((s, offset + len as usize))
575}
576
577#[cfg(test)]
578mod tests {
579    use super::*;
580
581    fn sample() -> TableDefLayout {
582        TableDefLayout {
583            version: 1,
584            name: "hosts".to_string(),
585            created_at: 0x0102_0304_0506_0708,
586            updated_at: 0x1112_1314_1516_1718,
587            columns: vec![
588                ColumnLayout {
589                    name: "id".to_string(),
590                    data_type: 2,
591                    nullable: false,
592                    default: None,
593                    vector_dim: None,
594                    compress: false,
595                    enum_variants: Vec::new(),
596                    decimal_precision: 4,
597                    element_type: None,
598                    metadata: vec![("desc".to_string(), "primary".to_string())],
599                },
600                ColumnLayout {
601                    name: "fingerprint".to_string(),
602                    data_type: 11,
603                    nullable: true,
604                    default: Some(vec![1, 2, 3]),
605                    vector_dim: Some(128),
606                    compress: true,
607                    enum_variants: vec!["a".to_string(), "b".to_string()],
608                    decimal_precision: 6,
609                    element_type: Some(4),
610                    metadata: Vec::new(),
611                },
612            ],
613            primary_key: vec!["id".to_string()],
614            indexes: vec![IndexLayout {
615                name: "idx_fp".to_string(),
616                index_type: 3,
617                unique: true,
618                columns: vec!["fingerprint".to_string()],
619            }],
620            constraints: vec![ConstraintLayout {
621                name: "fk_host".to_string(),
622                constraint_type: 3,
623                columns: vec!["host_id".to_string()],
624                ref_table: Some("hosts".to_string()),
625                ref_columns: Some(vec!["id".to_string()]),
626            }],
627        }
628    }
629
630    #[test]
631    fn round_trip_preserves_layout() {
632        let layout = sample();
633        let bytes = encode_table_def(&layout);
634        let decoded = decode_table_def(&bytes).expect("decode");
635        assert_eq!(decoded, layout);
636    }
637
638    #[test]
639    fn fixture_bytes_are_byte_identical() {
640        let layout = sample();
641        let bytes = encode_table_def(&layout);
642        assert_eq!(&bytes[0..4], b"RTBL", "magic must lead the payload");
643        assert_eq!(&bytes[4..8], &1u32.to_le_bytes(), "version field");
644        // name string: varint length (5) + "hosts"
645        assert_eq!(bytes[8], 5);
646        assert_eq!(&bytes[9..14], b"hosts");
647        // created_at u64 little-endian follows the name.
648        assert_eq!(&bytes[14..22], &0x0102_0304_0506_0708u64.to_le_bytes());
649    }
650
651    #[test]
652    fn rejects_short_and_bad_magic() {
653        assert_eq!(
654            decode_table_def(&[0u8; 2]),
655            Err(TableDefCodecError::TruncatedData)
656        );
657        let mut bytes = encode_table_def(&sample());
658        bytes[0] = b'X';
659        assert_eq!(
660            decode_table_def(&bytes),
661            Err(TableDefCodecError::InvalidMagic)
662        );
663    }
664}
reddb_file/table_def_codec.rs

reddb_file/
table_def_codec.rs
