reddb_file/
table_def.rs

1//! Persisted SQL table-definition payload codec (`RTBL`).
2//!
3//! The schema engine owns column types, index/constraint semantics, and
4//! validation. This module owns only the durable byte layout of a serialized
5//! table definition. Type discriminants (`data_type`, `element_type`,
6//! `index_type`, `constraint_type`) are carried as opaque `u8` bytes: the
7//! engine maps them to/from its `DataType` / `IndexType` / `ConstraintType`
8//! enums and is responsible for rejecting unknown discriminants.
9//!
10//! `reddb-file` already owns this table definition as the opaque
11//! `table_def_hex` field of `PhysicalCollectionContract`; this codec gives that
12//! hex blob a structured home.
13//!
14//! Strings are varint-length-prefixed UTF-8; counts are LEB128 varints; fixed
15//! integers are little-endian. DO NOT change magic/order/width — these bytes
16//! live in existing `.rdb` files.
17
18/// Magic prefix for a serialized table definition.
19pub const TABLE_DEF_MAGIC: [u8; 4] = *b"RTBL";
20
21/// A decoded column definition. Type bytes are opaque to this crate.
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub struct ColumnDefFrame {
24    pub name: String,
25    pub data_type: u8,
26    pub nullable: bool,
27    pub default: Option<Vec<u8>>,
28    pub vector_dim: Option<u32>,
29    pub compress: bool,
30    pub enum_variants: Vec<String>,
31    pub decimal_precision: u8,
32    pub element_type: Option<u8>,
33    /// Key/value metadata, in on-disk order.
34    pub metadata: Vec<(String, String)>,
35}
36
37/// A decoded index definition.
38#[derive(Debug, Clone, PartialEq, Eq)]
39pub struct IndexDefFrame {
40    pub name: String,
41    pub index_type: u8,
42    pub unique: bool,
43    pub columns: Vec<String>,
44}
45
46/// A decoded constraint definition.
47#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct ConstraintFrame {
49    pub name: String,
50    pub constraint_type: u8,
51    pub columns: Vec<String>,
52    pub ref_table: Option<String>,
53    pub ref_columns: Option<Vec<String>>,
54}
55
56/// A decoded table definition (`RTBL`) payload.
57#[derive(Debug, Clone, PartialEq, Eq)]
58pub struct TableDefFrame {
59    pub name: String,
60    pub version: u32,
61    pub created_at: u64,
62    pub updated_at: u64,
63    pub columns: Vec<ColumnDefFrame>,
64    pub primary_key: Vec<String>,
65    pub indexes: Vec<IndexDefFrame>,
66    pub constraints: Vec<ConstraintFrame>,
67}
68
69/// Errors decoding a table-definition payload.
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub enum TableDefFrameError {
72    TruncatedData,
73    InvalidMagic,
74    VarintOverflow,
75}
76
77impl std::fmt::Display for TableDefFrameError {
78    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
79        match self {
80            Self::TruncatedData => write!(f, "truncated data"),
81            Self::InvalidMagic => write!(f, "invalid magic bytes"),
82            Self::VarintOverflow => write!(f, "varint overflow"),
83        }
84    }
85}
86
87impl std::error::Error for TableDefFrameError {}
88
89// ============================================================================
90// Encode
91// ============================================================================
92
93/// Serialize a table-definition payload to bytes.
94pub fn encode_table_def_frame(frame: &TableDefFrame) -> Vec<u8> {
95    let mut buf = Vec::new();
96
97    buf.extend_from_slice(&TABLE_DEF_MAGIC);
98    buf.extend_from_slice(&frame.version.to_le_bytes());
99    write_string(&mut buf, &frame.name);
100    buf.extend_from_slice(&frame.created_at.to_le_bytes());
101    buf.extend_from_slice(&frame.updated_at.to_le_bytes());
102
103    write_varint(&mut buf, frame.columns.len() as u64);
104    for col in &frame.columns {
105        write_column(&mut buf, col);
106    }
107
108    write_varint(&mut buf, frame.primary_key.len() as u64);
109    for pk in &frame.primary_key {
110        write_string(&mut buf, pk);
111    }
112
113    write_varint(&mut buf, frame.indexes.len() as u64);
114    for idx in &frame.indexes {
115        write_index(&mut buf, idx);
116    }
117
118    write_varint(&mut buf, frame.constraints.len() as u64);
119    for constraint in &frame.constraints {
120        write_constraint(&mut buf, constraint);
121    }
122
123    buf
124}
125
126fn write_column(buf: &mut Vec<u8>, col: &ColumnDefFrame) {
127    write_string(buf, &col.name);
128    buf.push(col.data_type);
129    buf.push(if col.nullable { 1 } else { 0 });
130
131    if let Some(ref default) = col.default {
132        buf.push(1);
133        write_varint(buf, default.len() as u64);
134        buf.extend_from_slice(default);
135    } else {
136        buf.push(0);
137    }
138
139    if let Some(dim) = col.vector_dim {
140        buf.push(1);
141        buf.extend_from_slice(&dim.to_le_bytes());
142    } else {
143        buf.push(0);
144    }
145
146    buf.push(if col.compress { 1 } else { 0 });
147
148    write_varint(buf, col.enum_variants.len() as u64);
149    for variant in &col.enum_variants {
150        write_string(buf, variant);
151    }
152
153    buf.push(col.decimal_precision);
154
155    if let Some(et) = col.element_type {
156        buf.push(1);
157        buf.push(et);
158    } else {
159        buf.push(0);
160    }
161
162    write_varint(buf, col.metadata.len() as u64);
163    for (k, v) in &col.metadata {
164        write_string(buf, k);
165        write_string(buf, v);
166    }
167}
168
169fn write_index(buf: &mut Vec<u8>, idx: &IndexDefFrame) {
170    write_string(buf, &idx.name);
171    buf.push(idx.index_type);
172    buf.push(if idx.unique { 1 } else { 0 });
173    write_varint(buf, idx.columns.len() as u64);
174    for col in &idx.columns {
175        write_string(buf, col);
176    }
177}
178
179fn write_constraint(buf: &mut Vec<u8>, constraint: &ConstraintFrame) {
180    write_string(buf, &constraint.name);
181    buf.push(constraint.constraint_type);
182
183    write_varint(buf, constraint.columns.len() as u64);
184    for col in &constraint.columns {
185        write_string(buf, col);
186    }
187
188    if let Some(ref table) = constraint.ref_table {
189        buf.push(1);
190        write_string(buf, table);
191        if let Some(ref cols) = constraint.ref_columns {
192            write_varint(buf, cols.len() as u64);
193            for col in cols {
194                write_string(buf, col);
195            }
196        } else {
197            write_varint(buf, 0);
198        }
199    } else {
200        buf.push(0);
201    }
202}
203
204// ============================================================================
205// Decode
206// ============================================================================
207
208/// Deserialize a table-definition payload from bytes.
209pub fn decode_table_def_frame(data: &[u8]) -> Result<TableDefFrame, TableDefFrameError> {
210    if data.len() < 4 {
211        return Err(TableDefFrameError::TruncatedData);
212    }
213    if data[0..4] != TABLE_DEF_MAGIC {
214        return Err(TableDefFrameError::InvalidMagic);
215    }
216
217    let mut offset = 4;
218
219    if data.len() < offset + 4 {
220        return Err(TableDefFrameError::TruncatedData);
221    }
222    let version = u32::from_le_bytes(data[offset..offset + 4].try_into().expect("u32 checked"));
223    offset += 4;
224
225    let (name, name_len) = read_string(&data[offset..])?;
226    offset += name_len;
227
228    if data.len() < offset + 16 {
229        return Err(TableDefFrameError::TruncatedData);
230    }
231    let created_at = u64::from_le_bytes(data[offset..offset + 8].try_into().expect("u64 checked"));
232    offset += 8;
233    let updated_at = u64::from_le_bytes(data[offset..offset + 8].try_into().expect("u64 checked"));
234    offset += 8;
235
236    let (col_count, varint_len) = read_varint(&data[offset..])?;
237    offset += varint_len;
238    let mut columns = Vec::new();
239    for _ in 0..col_count {
240        let (col, col_len) = read_column(&data[offset..])?;
241        offset += col_len;
242        columns.push(col);
243    }
244
245    let (pk_count, varint_len) = read_varint(&data[offset..])?;
246    offset += varint_len;
247    let mut primary_key = Vec::new();
248    for _ in 0..pk_count {
249        let (pk, pk_len) = read_string(&data[offset..])?;
250        offset += pk_len;
251        primary_key.push(pk);
252    }
253
254    let (idx_count, varint_len) = read_varint(&data[offset..])?;
255    offset += varint_len;
256    let mut indexes = Vec::new();
257    for _ in 0..idx_count {
258        let (idx, idx_len) = read_index(&data[offset..])?;
259        offset += idx_len;
260        indexes.push(idx);
261    }
262
263    let (constraint_count, varint_len) = read_varint(&data[offset..])?;
264    offset += varint_len;
265    let mut constraints = Vec::new();
266    for _ in 0..constraint_count {
267        let (constraint, constraint_len) = read_constraint(&data[offset..])?;
268        offset += constraint_len;
269        constraints.push(constraint);
270    }
271
272    Ok(TableDefFrame {
273        name,
274        version,
275        created_at,
276        updated_at,
277        columns,
278        primary_key,
279        indexes,
280        constraints,
281    })
282}
283
284fn read_column(data: &[u8]) -> Result<(ColumnDefFrame, usize), TableDefFrameError> {
285    let mut offset = 0;
286
287    let (name, name_len) = read_string(&data[offset..])?;
288    offset += name_len;
289
290    if data.len() < offset + 2 {
291        return Err(TableDefFrameError::TruncatedData);
292    }
293    let data_type = data[offset];
294    offset += 1;
295    let nullable = data[offset] != 0;
296    offset += 1;
297
298    if data.len() < offset + 1 {
299        return Err(TableDefFrameError::TruncatedData);
300    }
301    let has_default = data[offset] != 0;
302    offset += 1;
303    let default = if has_default {
304        let (len, varint_len) = read_varint(&data[offset..])?;
305        offset += varint_len;
306        let len = usize::try_from(len).map_err(|_| TableDefFrameError::TruncatedData)?;
307        let end = offset
308            .checked_add(len)
309            .ok_or(TableDefFrameError::TruncatedData)?;
310        if data.len() < end {
311            return Err(TableDefFrameError::TruncatedData);
312        }
313        let default_data = data[offset..end].to_vec();
314        offset = end;
315        Some(default_data)
316    } else {
317        None
318    };
319
320    if data.len() < offset + 1 {
321        return Err(TableDefFrameError::TruncatedData);
322    }
323    let has_vector_dim = data[offset] != 0;
324    offset += 1;
325    let vector_dim = if has_vector_dim {
326        if data.len() < offset + 4 {
327            return Err(TableDefFrameError::TruncatedData);
328        }
329        let dim = u32::from_le_bytes(data[offset..offset + 4].try_into().expect("u32 checked"));
330        offset += 4;
331        Some(dim)
332    } else {
333        None
334    };
335
336    if data.len() < offset + 1 {
337        return Err(TableDefFrameError::TruncatedData);
338    }
339    let compress = data[offset] != 0;
340    offset += 1;
341
342    let (variant_count, varint_len) = read_varint(&data[offset..])?;
343    offset += varint_len;
344    let mut enum_variants = Vec::new();
345    for _ in 0..variant_count {
346        let (variant, variant_len) = read_string(&data[offset..])?;
347        offset += variant_len;
348        enum_variants.push(variant);
349    }
350
351    if data.len() < offset + 1 {
352        return Err(TableDefFrameError::TruncatedData);
353    }
354    let decimal_precision = data[offset];
355    offset += 1;
356
357    if data.len() < offset + 1 {
358        return Err(TableDefFrameError::TruncatedData);
359    }
360    let has_element_type = data[offset] != 0;
361    offset += 1;
362    let element_type = if has_element_type {
363        if data.len() < offset + 1 {
364            return Err(TableDefFrameError::TruncatedData);
365        }
366        let et = data[offset];
367        offset += 1;
368        Some(et)
369    } else {
370        None
371    };
372
373    let (meta_count, varint_len) = read_varint(&data[offset..])?;
374    offset += varint_len;
375    let mut metadata = Vec::new();
376    for _ in 0..meta_count {
377        let (k, k_len) = read_string(&data[offset..])?;
378        offset += k_len;
379        let (v, v_len) = read_string(&data[offset..])?;
380        offset += v_len;
381        metadata.push((k, v));
382    }
383
384    Ok((
385        ColumnDefFrame {
386            name,
387            data_type,
388            nullable,
389            default,
390            vector_dim,
391            compress,
392            enum_variants,
393            decimal_precision,
394            element_type,
395            metadata,
396        },
397        offset,
398    ))
399}
400
401fn read_index(data: &[u8]) -> Result<(IndexDefFrame, usize), TableDefFrameError> {
402    let mut offset = 0;
403
404    let (name, name_len) = read_string(&data[offset..])?;
405    offset += name_len;
406
407    if data.len() < offset + 2 {
408        return Err(TableDefFrameError::TruncatedData);
409    }
410    let index_type = data[offset];
411    offset += 1;
412    let unique = data[offset] != 0;
413    offset += 1;
414
415    let (col_count, varint_len) = read_varint(&data[offset..])?;
416    offset += varint_len;
417    let mut columns = Vec::new();
418    for _ in 0..col_count {
419        let (col, col_len) = read_string(&data[offset..])?;
420        offset += col_len;
421        columns.push(col);
422    }
423
424    Ok((
425        IndexDefFrame {
426            name,
427            index_type,
428            unique,
429            columns,
430        },
431        offset,
432    ))
433}
434
435fn read_constraint(data: &[u8]) -> Result<(ConstraintFrame, usize), TableDefFrameError> {
436    let mut offset = 0;
437
438    let (name, name_len) = read_string(&data[offset..])?;
439    offset += name_len;
440
441    if data.len() < offset + 1 {
442        return Err(TableDefFrameError::TruncatedData);
443    }
444    let constraint_type = data[offset];
445    offset += 1;
446
447    let (col_count, varint_len) = read_varint(&data[offset..])?;
448    offset += varint_len;
449    let mut columns = Vec::new();
450    for _ in 0..col_count {
451        let (col, col_len) = read_string(&data[offset..])?;
452        offset += col_len;
453        columns.push(col);
454    }
455
456    if data.len() < offset + 1 {
457        return Err(TableDefFrameError::TruncatedData);
458    }
459    let has_ref = data[offset] != 0;
460    offset += 1;
461
462    let (ref_table, ref_columns) = if has_ref {
463        let (table, table_len) = read_string(&data[offset..])?;
464        offset += table_len;
465
466        let (ref_col_count, varint_len) = read_varint(&data[offset..])?;
467        offset += varint_len;
468        let mut ref_cols = Vec::new();
469        for _ in 0..ref_col_count {
470            let (col, col_len) = read_string(&data[offset..])?;
471            offset += col_len;
472            ref_cols.push(col);
473        }
474
475        (Some(table), Some(ref_cols))
476    } else {
477        (None, None)
478    };
479
480    Ok((
481        ConstraintFrame {
482            name,
483            constraint_type,
484            columns,
485            ref_table,
486            ref_columns,
487        },
488        offset,
489    ))
490}
491
492// ============================================================================
493// Varint + string primitives (LEB128, varint-prefixed UTF-8)
494// ============================================================================
495
496fn write_varint(buf: &mut Vec<u8>, mut value: u64) {
497    loop {
498        let mut byte = (value & 0x7F) as u8;
499        value >>= 7;
500        if value != 0 {
501            byte |= 0x80;
502        }
503        buf.push(byte);
504        if value == 0 {
505            break;
506        }
507    }
508}
509
510fn read_varint(data: &[u8]) -> Result<(u64, usize), TableDefFrameError> {
511    let mut result: u64 = 0;
512    let mut shift = 0;
513    let mut offset = 0;
514
515    loop {
516        if offset >= data.len() {
517            return Err(TableDefFrameError::TruncatedData);
518        }
519        let byte = data[offset];
520        offset += 1;
521
522        if shift >= 64 {
523            return Err(TableDefFrameError::VarintOverflow);
524        }
525
526        result |= ((byte & 0x7F) as u64) << shift;
527        shift += 7;
528
529        if byte & 0x80 == 0 {
530            break;
531        }
532    }
533
534    Ok((result, offset))
535}
536
537fn write_string(buf: &mut Vec<u8>, s: &str) {
538    let bytes = s.as_bytes();
539    write_varint(buf, bytes.len() as u64);
540    buf.extend_from_slice(bytes);
541}
542
543fn read_string(data: &[u8]) -> Result<(String, usize), TableDefFrameError> {
544    let (len, varint_len) = read_varint(data)?;
545    let offset = varint_len;
546    let len = usize::try_from(len).map_err(|_| TableDefFrameError::TruncatedData)?;
547    let end = offset
548        .checked_add(len)
549        .ok_or(TableDefFrameError::TruncatedData)?;
550    if data.len() < end {
551        return Err(TableDefFrameError::TruncatedData);
552    }
553    let s = String::from_utf8(data[offset..end].to_vec())
554        .map_err(|_| TableDefFrameError::TruncatedData)?;
555    Ok((s, end))
556}
557
558#[cfg(test)]
559mod tests {
560    use super::*;
561
562    fn sample_frame() -> TableDefFrame {
563        TableDefFrame {
564            name: "embeddings".into(),
565            version: 1,
566            created_at: 1_700_000_000,
567            updated_at: 1_700_000_500,
568            columns: vec![
569                ColumnDefFrame {
570                    name: "id".into(),
571                    data_type: 2,
572                    nullable: false,
573                    default: None,
574                    vector_dim: None,
575                    compress: false,
576                    enum_variants: vec![],
577                    decimal_precision: 4,
578                    element_type: None,
579                    metadata: vec![],
580                },
581                ColumnDefFrame {
582                    name: "embedding".into(),
583                    data_type: 11,
584                    nullable: false,
585                    default: Some(vec![1, 2, 3]),
586                    vector_dim: Some(384),
587                    compress: true,
588                    enum_variants: vec!["a".into(), "b".into()],
589                    decimal_precision: 6,
590                    element_type: Some(3),
591                    metadata: vec![("unit".into(), "f32".into())],
592                },
593            ],
594            primary_key: vec!["id".into()],
595            indexes: vec![IndexDefFrame {
596                name: "idx_vec".into(),
597                index_type: 4,
598                unique: false,
599                columns: vec!["embedding".into()],
600            }],
601            constraints: vec![
602                ConstraintFrame {
603                    name: "fk".into(),
604                    constraint_type: 3,
605                    columns: vec!["id".into()],
606                    ref_table: Some("other".into()),
607                    ref_columns: Some(vec!["oid".into()]),
608                },
609                ConstraintFrame {
610                    name: "nn".into(),
611                    constraint_type: 5,
612                    columns: vec!["id".into()],
613                    ref_table: None,
614                    ref_columns: None,
615                },
616            ],
617        }
618    }
619
620    #[test]
621    fn table_def_frame_round_trips() {
622        let frame = sample_frame();
623        let encoded = encode_table_def_frame(&frame);
624        let decoded = decode_table_def_frame(&encoded).unwrap();
625        assert_eq!(decoded, frame);
626        assert_eq!(encode_table_def_frame(&decoded), encoded);
627    }
628
629    #[test]
630    fn table_def_frame_pins_magic_and_version() {
631        let encoded = encode_table_def_frame(&sample_frame());
632        assert_eq!(&encoded[0..4], b"RTBL");
633        assert_eq!(&encoded[4..8], &1u32.to_le_bytes());
634    }
635
636    #[test]
637    fn table_def_frame_rejects_bad_input() {
638        assert_eq!(
639            decode_table_def_frame(&[0u8; 2]),
640            Err(TableDefFrameError::TruncatedData)
641        );
642        let mut bad = encode_table_def_frame(&sample_frame());
643        bad[0] = b'X';
644        assert_eq!(
645            decode_table_def_frame(&bad),
646            Err(TableDefFrameError::InvalidMagic)
647        );
648        let encoded = encode_table_def_frame(&sample_frame());
649        assert_eq!(
650            decode_table_def_frame(&encoded[..encoded.len() - 1]),
651            Err(TableDefFrameError::TruncatedData)
652        );
653    }
654
655    #[test]
656    fn table_def_frame_does_not_preallocate_untrusted_counts() {
657        let mut encoded = Vec::new();
658        encoded.extend_from_slice(&TABLE_DEF_MAGIC);
659        encoded.extend_from_slice(&1u32.to_le_bytes());
660        write_string(&mut encoded, "t");
661        encoded.extend_from_slice(&0u64.to_le_bytes());
662        encoded.extend_from_slice(&0u64.to_le_bytes());
663        write_varint(&mut encoded, u64::MAX);
664
665        assert_eq!(
666            decode_table_def_frame(&encoded),
667            Err(TableDefFrameError::TruncatedData)
668        );
669    }
670}
reddb_file/table_def.rs

reddb_file/
table_def.rs
