verit_core/

idl.rs

//! The `.vsc` schema IDL — a small text front-end that compiles to a
//! [`Schema`]. It invents no wire semantics: it drives [`SchemaBuilder`], so
//! the result is exactly the same canonical `VSC1` bytes (and 128-bit id) any
//! other definition of the same schema produces. The IDL therefore cannot
//! drift from the wire format — the id is the contract, and it is computed the
//! same way regardless of how the schema was written.
//!
//! ## Grammar (v1)
//!
//! ```text
//! // line comments
//! struct Order {                 // sparse struct (presence bitmap)
//!   1: id     u64                //   <field-id> : <name> <type>
//!   2: item   string
//!   3: qty    u32
//!   4: tags   list<string>
//!   5: origin Point              // reference to a named struct
//!   6: level  Level              // reference to a named enum
//! }
//!
//! dense struct Point { 1: x f64  2: y f64 }        // all fields mandatory, no bitmap
//! packed struct Wide { 1: a u8  2: b u64 }         // present-only slots, popcount-indexed
//!
//! enum Level { 0: Debug  1: Info  2: Error }       // open, u32 repr
//!
//! root Order                     // the message root type (required, exactly one)
//! ```
//!
//! Field IDs are explicit and are the evolution contract — never reuse an ID
//! for a new meaning. Scalars: `bool`, `u8`..`u64`, `i8`..`i64`, `f32`, `f64`,
//! `string`, `bytes`. Compound: `list<T>` and `map<K, V>` (both nestable) and a
//! bare type name referring to a declared `struct`/`enum`. Map keys must be a
//! `bool`, an integer, `string`, or an enum.

use crate::error::{Error, Result};
use crate::schema::{Dt, Schema, SchemaBuilder};
use crate::value::Value;

fn err(msg: impl Into<String>) -> Error {
    Error::BadSchema(format!("idl: {}", msg.into()))
}

// ---------------------------------------------------------------------------
// Tokenizer
// ---------------------------------------------------------------------------

#[derive(Debug, PartialEq)]
enum Tok {
    Ident(String), // keywords and names both arrive as idents
    Int(u64),
    LBrace,
    RBrace,
    Lt,
    Gt,
    Colon,
    Eq,
    Minus,
}

fn tokenize(src: &str) -> Result<Vec<Tok>> {
    let b = src.as_bytes();
    let mut i = 0;
    let mut out = Vec::new();
    while i < b.len() {
        let c = b[i];
        match c {
            b' ' | b'\t' | b'\r' | b'\n' | b',' => i += 1, // whitespace + optional commas
            b'/' if i + 1 < b.len() && b[i + 1] == b'/' => {
                while i < b.len() && b[i] != b'\n' {
                    i += 1;
                }
            }
            b'{' => {
                out.push(Tok::LBrace);
                i += 1;
            }
            b'}' => {
                out.push(Tok::RBrace);
                i += 1;
            }
            b'<' => {
                out.push(Tok::Lt);
                i += 1;
            }
            b'>' => {
                out.push(Tok::Gt);
                i += 1;
            }
            b':' => {
                out.push(Tok::Colon);
                i += 1;
            }
            b'=' => {
                out.push(Tok::Eq);
                i += 1;
            }
            b'-' => {
                out.push(Tok::Minus);
                i += 1;
            }
            c if c.is_ascii_digit() => {
                let start = i;
                while i < b.len() && b[i].is_ascii_digit() {
                    i += 1;
                }
                let n: u64 = src[start..i]
                    .parse()
                    .map_err(|_| err(format!("number too large: {}", &src[start..i])))?;
                out.push(Tok::Int(n));
            }
            c if c.is_ascii_alphabetic() || c == b'_' => {
                let start = i;
                while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
                    i += 1;
                }
                out.push(Tok::Ident(src[start..i].to_string()));
            }
            other => return Err(err(format!("unexpected character {:?}", other as char))),
        }
    }
    Ok(out)
}

// ---------------------------------------------------------------------------
// Parser (owned AST so field-name refs outlive the SchemaBuilder calls)
// ---------------------------------------------------------------------------

enum Mode {
    Sparse,
    Dense,
    Packed,
}

struct StructAst {
    name: String,
    mode: Mode,
    fields: Vec<(u16, String, Dt)>,
    defaults: Vec<(u16, Value)>,
}
struct EnumAst {
    name: String,
    variants: Vec<(u32, String)>,
}

struct Parser<'t> {
    toks: &'t [Tok],
    pos: usize,
}

impl<'t> Parser<'t> {
    fn peek(&self) -> Option<&'t Tok> {
        self.toks.get(self.pos)
    }
    fn next(&mut self) -> Result<&'t Tok> {
        let t = self
            .toks
            .get(self.pos)
            .ok_or_else(|| err("unexpected end of input"))?;
        self.pos += 1;
        Ok(t)
    }
    fn ident(&mut self) -> Result<String> {
        match self.next()? {
            Tok::Ident(s) => Ok(s.clone()),
            other => Err(err(format!("expected a name, found {other:?}"))),
        }
    }
    fn expect(&mut self, want: &Tok) -> Result<()> {
        let got = self.next()?;
        if got == want {
            Ok(())
        } else {
            Err(err(format!("expected {want:?}, found {got:?}")))
        }
    }

    /// A type expression: `list<T>`, a scalar keyword, or a named ref.
    fn type_expr(&mut self) -> Result<Dt> {
        let name = self.ident()?;
        if name == "list" {
            self.expect(&Tok::Lt)?;
            let elem = self.type_expr()?;
            self.expect(&Tok::Gt)?;
            return Ok(Dt::list(elem));
        }
        if name == "map" {
            // `map<K, V>` — the comma is optional (commas are whitespace).
            self.expect(&Tok::Lt)?;
            let key = self.type_expr()?;
            let value = self.type_expr()?;
            self.expect(&Tok::Gt)?;
            return Ok(Dt::map(key, value));
        }
        if name == "union" {
            // `union<T0, T1, …>` — one or more variant types (commas optional).
            self.expect(&Tok::Lt)?;
            let mut variants = Vec::new();
            while self.peek() != Some(&Tok::Gt) {
                variants.push(self.type_expr()?);
            }
            self.expect(&Tok::Gt)?;
            return Ok(Dt::union(variants));
        }
        Ok(match name.as_str() {
            "bool" => Dt::Bool,
            "u8" => Dt::U8,
            "u16" => Dt::U16,
            "u32" => Dt::U32,
            "u64" => Dt::U64,
            "i8" => Dt::I8,
            "i16" => Dt::I16,
            "i32" => Dt::I32,
            "i64" => Dt::I64,
            "f32" => Dt::F32,
            "f64" => Dt::F64,
            "string" | "str" => Dt::Str,
            "bytes" => Dt::Bytes,
            // Anything else is a reference to a declared struct/enum.
            _ => Dt::named(&name),
        })
    }

    fn struct_def(&mut self, mode: Mode) -> Result<StructAst> {
        let name = self.ident()?;
        self.expect(&Tok::LBrace)?;
        let mut fields = Vec::new();
        let mut defaults = Vec::new();
        while self.peek() != Some(&Tok::RBrace) {
            let id = match self.next()? {
                Tok::Int(n) if *n <= u16::MAX as u64 => *n as u16,
                Tok::Int(n) => return Err(err(format!("field id {n} exceeds u16 in {name}"))),
                other => {
                    return Err(err(format!(
                        "expected a field id in {name}, found {other:?}"
                    )))
                }
            };
            self.expect(&Tok::Colon)?;
            let fname = self.ident()?;
            let ty = self.type_expr()?;
            // Optional `= <literal>` custom default (scalar fields only).
            if self.peek() == Some(&Tok::Eq) {
                self.next()?;
                defaults.push((id, self.default_literal(&ty)?));
            }
            fields.push((id, fname, ty));
        }
        self.expect(&Tok::RBrace)?;
        Ok(StructAst {
            name,
            mode,
            fields,
            defaults,
        })
    }

    /// A scalar default literal (`= 5`, `= -3`, `= true`), typed by the field's
    /// declared type `ty`.
    fn default_literal(&mut self, ty: &Dt) -> Result<Value> {
        if let Some(Tok::Ident(s)) = self.peek() {
            if s == "true" || s == "false" {
                let b = s == "true";
                self.next()?;
                return Ok(Value::Bool(b));
            }
        }
        let neg = if self.peek() == Some(&Tok::Minus) {
            self.next()?;
            true
        } else {
            false
        };
        let n = match self.next()? {
            Tok::Int(n) => *n,
            other => return Err(err(format!("expected a default literal, found {other:?}"))),
        };
        let s = |n: u64| -> i64 {
            if neg {
                -(n as i64)
            } else {
                n as i64
            }
        };
        Ok(match ty {
            Dt::U8 => Value::U8(n as u8),
            Dt::U16 => Value::U16(n as u16),
            Dt::U32 => Value::U32(n as u32),
            Dt::U64 => Value::U64(n),
            Dt::I8 => Value::I8(s(n) as i8),
            Dt::I16 => Value::I16(s(n) as i16),
            Dt::I32 => Value::I32(s(n) as i32),
            Dt::I64 => Value::I64(s(n)),
            Dt::F32 => Value::F32(if neg { -(n as f32) } else { n as f32 }),
            Dt::F64 => Value::F64(if neg { -(n as f64) } else { n as f64 }),
            // A named type here is validated as an enum at build time.
            Dt::Named(_) => Value::Enum(n as u32),
            _ => return Err(err("a default is only allowed on a scalar field")),
        })
    }

    fn enum_def(&mut self) -> Result<EnumAst> {
        let name = self.ident()?;
        self.expect(&Tok::LBrace)?;
        let mut variants = Vec::new();
        while self.peek() != Some(&Tok::RBrace) {
            let value = match self.next()? {
                Tok::Int(n) if *n <= u32::MAX as u64 => *n as u32,
                Tok::Int(n) => return Err(err(format!("enum value {n} exceeds u32 in {name}"))),
                other => {
                    return Err(err(format!(
                        "expected an enum value in {name}, found {other:?}"
                    )))
                }
            };
            self.expect(&Tok::Colon)?;
            let vname = self.ident()?;
            variants.push((value, vname));
        }
        self.expect(&Tok::RBrace)?;
        Ok(EnumAst { name, variants })
    }
}

/// Parse `.vsc` IDL source into a [`Schema`]. Errors are `BadSchema` with an
/// `idl:` prefix and a human-readable message.
pub fn parse(src: &str) -> Result<Schema> {
    let toks = tokenize(src)?;
    let mut p = Parser {
        toks: &toks,
        pos: 0,
    };

    let mut structs: Vec<StructAst> = Vec::new();
    let mut enums: Vec<EnumAst> = Vec::new();
    let mut root: Option<String> = None;

    while let Some(t) = p.peek() {
        let kw = match t {
            Tok::Ident(s) => s.clone(),
            other => {
                return Err(err(format!(
                    "expected a top-level declaration, found {other:?}"
                )))
            }
        };
        p.pos += 1; // consume the keyword
        match kw.as_str() {
            "struct" => structs.push(p.struct_def(Mode::Sparse)?),
            "dense" => {
                if p.ident()? != "struct" {
                    return Err(err("`dense` must be followed by `struct`"));
                }
                structs.push(p.struct_def(Mode::Dense)?);
            }
            "packed" => {
                if p.ident()? != "struct" {
                    return Err(err("`packed` must be followed by `struct`"));
                }
                structs.push(p.struct_def(Mode::Packed)?);
            }
            "enum" => enums.push(p.enum_def()?),
            "root" => {
                if root.is_some() {
                    return Err(err("more than one `root` declaration"));
                }
                root = Some(p.ident()?);
            }
            other => return Err(err(format!("unknown top-level keyword `{other}`"))),
        }
    }

    let root = root.ok_or_else(|| err("missing `root <TypeName>` declaration"))?;

    // Build the schema. Field-name refs borrow the AST, which outlives this.
    let mut b = SchemaBuilder::new();
    for s in &structs {
        let fields: Vec<(u16, &str, Dt)> = s
            .fields
            .iter()
            .map(|(id, n, ty)| (*id, n.as_str(), ty.clone()))
            .collect();
        b = match s.mode {
            Mode::Sparse => b.add_struct(&s.name, fields),
            Mode::Dense => b.add_dense_struct(&s.name, fields),
            Mode::Packed => b.add_packed_struct(&s.name, fields),
        };
    }
    for e in &enums {
        let variants: Vec<(u32, &str)> = e.variants.iter().map(|(v, n)| (*v, n.as_str())).collect();
        b = b.add_enum(&e.name, variants);
    }
    for s in &structs {
        for (fid, value) in &s.defaults {
            b = b.set_default(&s.name, *fid, value.clone());
        }
    }
    b.build(&root)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn idl_matches_hand_built_schema_id() {
        let src = r#"
            // an order record
            dense struct Point { 1: x f64  2: y f64 }
            enum Level { 0: Debug  1: Info  2: Error }
            struct Order {
                1: id     u64
                2: item   string
                3: qty    u32
                4: tags   list<string>
                5: origin Point
                6: level  Level
                7: grid   list<list<u16>>
            }
            root Order
        "#;
        let from_idl = parse(src).unwrap();

        let hand = SchemaBuilder::new()
            .add_dense_struct("Point", vec![(1, "x", Dt::F64), (2, "y", Dt::F64)])
            .add_enum("Level", vec![(0, "Debug"), (1, "Info"), (2, "Error")])
            .add_struct(
                "Order",
                vec![
                    (1, "id", Dt::U64),
                    (2, "item", Dt::Str),
                    (3, "qty", Dt::U32),
                    (4, "tags", Dt::list(Dt::Str)),
                    (5, "origin", Dt::named("Point")),
                    (6, "level", Dt::named("Level")),
                    (7, "grid", Dt::list(Dt::list(Dt::U16))),
                ],
            )
            .build("Order")
            .unwrap();

        // The IDL is a front-end to the canonical schema: same id, byte-for-byte.
        assert_eq!(from_idl.id(), hand.id());
        assert_eq!(from_idl.canonical_bytes(), hand.canonical_bytes());
    }

    #[test]
    fn declaration_order_is_irrelevant() {
        let a = parse("struct A { 1: x u8 } root A").unwrap();
        let b = parse("  root A\nstruct A {1:x u8}").unwrap();
        assert_eq!(a.id(), b.id());
    }

    #[test]
    fn errors_are_typed_not_panics() {
        assert!(parse("struct {").is_err()); // no name
        assert!(parse("struct A { 1 x u8 } root A").is_err()); // missing colon
        assert!(parse("struct A { 1: x u8 }").is_err()); // no root
        assert!(parse("root Missing").is_err()); // root refers to nothing
        assert!(parse("struct A { 99999999: x u8 } root A").is_err()); // field id > u16
    }
}