datumesh_core/
lexical.rs

1//! Lexical forms: names (2), scalars (3), hex, licences, key references (9.1) and units (7.1).
2//!
3//! Every check takes the path of the value it checks, for the error it may return. Each value
4//! is checked against its lexical form first and its length limit second (SPEC 4.5).
5
6use crate::error::{err, ErrorCode, Result};
7use crate::tables;
8
9/// Limits of SPEC 4.5.
10pub mod limits {
11    /// Characters in a function name, path segment, argument name or code language.
12    pub const NAME: usize = 64;
13    /// Characters in `<namespace>/<function>`.
14    pub const FUNCTION_NAME: usize = 512;
15    /// Arguments per function or cell.
16    pub const ARGS: usize = 32;
17    /// Characters in a number.
18    pub const NUMBER: usize = 128;
19    /// UTF-8 bytes in a string value.
20    pub const STRING: usize = 1024;
21    /// UTF-8 bytes in a receipt summary.
22    pub const SUMMARY: usize = 1024;
23    /// UTF-8 bytes in a function body.
24    pub const BODY: usize = 1_048_576;
25    /// Entries in a receipt's or descriptor's `inputs`.
26    pub const INPUTS: usize = 4096;
27    /// Characters in an SPDX licence identifier.
28    pub const LICENCE_SPDX: usize = 64;
29    /// Characters in a licence URL.
30    pub const LICENCE_URL: usize = 512;
31    /// Keys in one key publication.
32    pub const KEYS: usize = 64;
33    /// Characters in a blob media type (3.7).
34    pub const MEDIA_TYPE: usize = 128;
35    /// Characters in an index unit (`units.md` 2.4).
36    pub const INDEX_UNIT: usize = 64;
37    /// The largest blob length, 2^53 - 1.
38    pub const BLOB_LENGTH: u64 = 9_007_199_254_740_991;
39    /// Values in an argument's extent (7; `spec-0.4.0`).
40    pub const EXTENT: usize = 4096;
41    /// Member receipt hashes in a slice manifest (15.13; `spec-0.4.0`).
42    pub const MEMBERS: usize = 1_048_576;
43}
44
45fn is_lower_digit(c: u8) -> bool {
46    c.is_ascii_lowercase() || c.is_ascii_digit()
47}
48
49/// The grammar of 2.2 without its length limit.
50pub(crate) fn is_name_grammar(s: &str) -> bool {
51    let b = s.as_bytes();
52    match b.first() {
53        Some(c) if c.is_ascii_lowercase() => {}
54        _ => return false,
55    }
56    let mut prev_underscore = false;
57    for &c in &b[1..] {
58        if c == b'_' {
59            if prev_underscore {
60                return false;
61            }
62            prev_underscore = true;
63        } else if is_lower_digit(c) {
64            prev_underscore = false;
65        } else {
66            return false;
67        }
68    }
69    !prev_underscore
70}
71
72/// A name (2.2): function names, path segments and code languages. `E_CANON_NAME`, then `E_CANON_LIMIT`.
73pub fn check_name(s: &str, path: &str) -> Result<()> {
74    if !is_name_grammar(s) {
75        return err(ErrorCode::CanonName, path, format!("not a lowercase snake_case name: {s:?}"));
76    }
77    if s.len() > limits::NAME {
78        return err(ErrorCode::CanonLimit, path, "name longer than 64 characters");
79    }
80    Ok(())
81}
82
83/// An argument name (2.4): grammar (`E_CANON_ARG_NAME`), length (`E_CANON_LIMIT`), then reserved names.
84pub fn check_arg_name(s: &str, path: &str) -> Result<()> {
85    if !is_name_grammar(s) {
86        return err(ErrorCode::CanonArgName, path, format!("not a lowercase snake_case name: {s:?}"));
87    }
88    if s.len() > limits::NAME {
89        return err(ErrorCode::CanonLimit, path, "name longer than 64 characters");
90    }
91    if s == "asof" || s.starts_with("dm_") {
92        return err(ErrorCode::CanonArgName, path, format!("reserved argument name {s:?}"));
93    }
94    Ok(())
95}
96
97/// A DNS name (2.1). Every failure is `E_CANON_NAME`.
98pub fn check_dns_name(s: &str, path: &str) -> Result<()> {
99    if s.len() > 253 {
100        return err(ErrorCode::CanonName, path, "DNS name longer than 253 characters");
101    }
102    let labels: Vec<&str> = s.split('.').collect();
103    if labels.len() < 2 {
104        return err(ErrorCode::CanonName, path, format!("DNS name needs at least two labels: {s:?}"));
105    }
106    for l in &labels {
107        let b = l.as_bytes();
108        let ok = !b.is_empty()
109            && b.len() <= 63
110            && b.iter().all(|&c| is_lower_digit(c) || c == b'-')
111            && b[0] != b'-'
112            && b[b.len() - 1] != b'-';
113        if !ok {
114            return err(ErrorCode::CanonName, path, format!("bad DNS label {l:?}"));
115        }
116    }
117    if labels.last().is_some_and(|l| l.bytes().all(|c| c.is_ascii_digit())) {
118        return err(ErrorCode::CanonName, path, "top-level label is numeric");
119    }
120    Ok(())
121}
122
123/// A namespace (2.3): a DNS name, then path segments that are names.
124pub fn check_namespace(s: &str, path: &str) -> Result<()> {
125    let mut parts = s.split('/');
126    check_dns_name(parts.next().unwrap_or(""), path)?;
127    for seg in parts {
128        check_name(seg, path)?;
129    }
130    Ok(())
131}
132
133/// The host of a namespace: its DNS name.
134pub(crate) fn host_of(namespace: &str) -> &str {
135    namespace.split('/').next().unwrap_or(namespace)
136}
137
138/// A fully qualified function name (2.3): namespace, `/`, name; at most 512 characters.
139pub fn check_function_name(s: &str, path: &str) -> Result<()> {
140    let Some((ns, f)) = s.rsplit_once('/') else {
141        return err(ErrorCode::CanonName, path, format!("function name has no namespace: {s:?}"));
142    };
143    check_namespace(ns, path)?;
144    check_name(f, path)?;
145    if s.len() > limits::FUNCTION_NAME {
146        return err(ErrorCode::CanonLimit, path, "function name longer than 512 characters");
147    }
148    Ok(())
149}
150
151/// Lowercase hex of exactly `digits` digits (`E_CANON_HEX`).
152pub fn check_hex(s: &str, digits: usize, path: &str) -> Result<()> {
153    if s.len() != digits || !s.bytes().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) {
154        return err(ErrorCode::CanonHex, path, format!("expected {digits} lowercase hex digits"));
155    }
156    Ok(())
157}
158
159/// A key reference `<host>#<key id>` (9.1).
160pub fn check_key_ref(s: &str, path: &str) -> Result<()> {
161    let Some((host, id)) = s.split_once('#') else {
162        return err(ErrorCode::CanonName, path, "a key reference is <host>#<key id>");
163    };
164    check_dns_name(host, path)?;
165    check_hex(id, 16, path)
166}
167
168/// The number grammar of 3.1 without the length limit.
169pub(crate) fn is_number_grammar(s: &str) -> bool {
170    let b = s.as_bytes();
171    let mut i = 0;
172    if b.first() == Some(&b'-') {
173        i = 1;
174    }
175    match b.get(i) {
176        Some(b'0') => i += 1,
177        Some(b'1'..=b'9') => {
178            while b.get(i).is_some_and(u8::is_ascii_digit) {
179                i += 1;
180            }
181        }
182        _ => return false,
183    }
184    if b.get(i) == Some(&b'.') {
185        let frac = &b[i + 1..];
186        if frac.is_empty() || !frac.iter().all(u8::is_ascii_digit) || frac[frac.len() - 1] == b'0' {
187            return false;
188        }
189        i = b.len();
190    }
191    i == b.len() && s != "-0"
192}
193
194/// Compares two numbers in canonical form (3.1) numerically.
195pub(crate) fn cmp_numbers(a: &str, b: &str) -> core::cmp::Ordering {
196    use core::cmp::Ordering;
197    let (an, a) = a.strip_prefix('-').map_or((false, a), |x| (true, x));
198    let (bn, b) = b.strip_prefix('-').map_or((false, b), |x| (true, x));
199    // Canonical: no "-0", so a negative number is below every other.
200    let magnitude = || {
201        let (ai, af) = a.split_once('.').unwrap_or((a, ""));
202        let (bi, bf) = b.split_once('.').unwrap_or((b, ""));
203        ai.len().cmp(&bi.len()).then_with(|| ai.cmp(bi)).then_with(|| af.cmp(bf))
204    };
205    match (an, bn) {
206        (false, false) => magnitude(),
207        (true, true) => magnitude().reverse(),
208        (true, false) => Ordering::Less,
209        (false, true) => Ordering::Greater,
210    }
211}
212
213/// Whether a canonical number is an integer.
214pub(crate) fn is_integer(s: &str) -> bool {
215    !s.contains('.')
216}
217
218/// Whether `s` is an `inputs` entry of the slice form `slice:<64 hex>:<64 hex>` (8.1).
219pub(crate) fn split_slice_entry(s: &str) -> Option<(&str, &str)> {
220    let rest = s.strip_prefix("slice:")?;
221    let (id, members) = rest.split_once(':')?;
222    let hex = |h: &str| h.len() == 64 && h.bytes().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c));
223    (hex(id) && hex(members)).then_some((id, members))
224}
225
226/// An `inputs` entry (8.1, 10.1 step 3): 64 lowercase hex digits, or since `spec-0.4.0` a
227/// slice entry `slice:<slice id>:<members hash>` (135 characters); `E_CANON_HEX` otherwise.
228pub fn check_input_entry(s: &str, path: &str) -> Result<()> {
229    if s.starts_with("slice:") {
230        if split_slice_entry(s).is_none() {
231            return err(ErrorCode::CanonHex, path, format!("not a slice entry slice:<64 hex>:<64 hex>: {s:?}"));
232        }
233        return Ok(());
234    }
235    check_hex(s, 64, path)
236}
237
238/// A number (3.1): `E_CANON_NUMBER`, then `E_CANON_LIMIT`.
239pub fn check_number(s: &str, path: &str) -> Result<()> {
240    if !is_number_grammar(s) {
241        return err(ErrorCode::CanonNumber, path, format!("not a canonical decimal: {s:?}"));
242    }
243    if s.len() > limits::NUMBER {
244        return err(ErrorCode::CanonLimit, path, "number longer than 128 characters");
245    }
246    Ok(())
247}
248
249fn digits(b: &[u8]) -> Option<u32> {
250    if b.is_empty() || !b.iter().all(u8::is_ascii_digit) {
251        return None;
252    }
253    Some(b.iter().fold(0u32, |a, &c| a * 10 + u32::from(c - b'0')))
254}
255
256fn days_in_month(y: u32, m: u32) -> u32 {
257    match m {
258        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
259        4 | 6 | 9 | 11 => 30,
260        2 if (y.is_multiple_of(4) && !y.is_multiple_of(100)) || y.is_multiple_of(400) => 29,
261        2 => 28,
262        _ => 0,
263    }
264}
265
266/// `YYYY-MM-DD`, a real day in years 0001-9999.
267pub(crate) fn is_date(s: &str) -> bool {
268    let b = s.as_bytes();
269    if b.len() != 10 || b[4] != b'-' || b[7] != b'-' {
270        return false;
271    }
272    let (Some(y), Some(m), Some(d)) = (digits(&b[0..4]), digits(&b[5..7]), digits(&b[8..10])) else {
273        return false;
274    };
275    (1..=9999).contains(&y) && (1..=12).contains(&m) && d >= 1 && d <= days_in_month(y, m)
276}
277
278/// A date (3.2), `E_CANON_DATE`.
279pub fn check_date(s: &str, path: &str) -> Result<()> {
280    if !is_date(s) {
281        return err(ErrorCode::CanonDate, path, format!("not a YYYY-MM-DD calendar date: {s:?}"));
282    }
283    Ok(())
284}
285
286/// `YYYY-MM-DDTHH:MM:SS.ffffffZ`.
287pub(crate) fn is_instant(s: &str) -> bool {
288    let b = s.as_bytes();
289    if b.len() != 27 || !s.is_char_boundary(10) || !is_date(&s[..10]) {
290        return false;
291    }
292    if b[10] != b'T' || b[13] != b':' || b[16] != b':' || b[19] != b'.' || b[26] != b'Z' {
293        return false;
294    }
295    let (Some(h), Some(mi), Some(se), Some(_)) =
296        (digits(&b[11..13]), digits(&b[14..16]), digits(&b[17..19]), digits(&b[20..26]))
297    else {
298        return false;
299    };
300    h < 24 && mi < 60 && se < 60
301}
302
303/// An instant (3.3), `E_CANON_INSTANT`.
304pub fn check_instant(s: &str, path: &str) -> Result<()> {
305    if !is_instant(s) {
306        return err(ErrorCode::CanonInstant, path, format!("not YYYY-MM-DDTHH:MM:SS.ffffffZ: {s:?}"));
307    }
308    Ok(())
309}
310
311fn is_restricted_name(s: &str) -> bool {
312    let b = s.as_bytes();
313    !b.is_empty() && is_lower_digit(b[0]) && b.iter().all(|&c| is_lower_digit(c) || b"!#$&-^_.+".contains(&c))
314}
315
316/// A blob (3.7): its sha256, length and media type. Every failure is `E_CANON_BLOB`.
317pub(crate) fn parse_blob(s: &str) -> core::result::Result<([u8; 32], u64, &str), String> {
318    let mut parts = s.split(':');
319    let (Some(h), Some(len), Some(mt), None) = (parts.next(), parts.next(), parts.next(), parts.next()) else {
320        return Err("a blob is <sha256>:<length>:<media type>".into());
321    };
322    let hash = match decode_hex_lower::<32>(h) {
323        Some(x) => x,
324        None => return Err("the hash is 64 lowercase hex digits".into()),
325    };
326    let lb = len.as_bytes();
327    if lb.is_empty() || !lb.iter().all(u8::is_ascii_digit) || (lb.len() > 1 && lb[0] == b'0') || lb.len() > 16 {
328        return Err("the length is a canonical non-negative integer".into());
329    }
330    let n: u64 = len.parse().map_err(|_| String::from("bad length"))?;
331    if n > limits::BLOB_LENGTH {
332        return Err("length above 9007199254740991".into());
333    }
334    let ok_mt = match mt.split_once('/') {
335        Some((t, st)) => is_restricted_name(t) && is_restricted_name(st),
336        None => false,
337    };
338    if !ok_mt {
339        return Err(format!("not a lowercase type/subtype media type: {mt:?}"));
340    }
341    if mt.len() > limits::MEDIA_TYPE {
342        return Err("media type longer than 128 characters".into());
343    }
344    Ok((hash, n, mt))
345}
346
347/// A blob (3.7), `E_CANON_BLOB`.
348pub fn check_blob(s: &str, path: &str) -> Result<()> {
349    match parse_blob(s) {
350        Ok(_) => Ok(()),
351        Err(d) => err(ErrorCode::CanonBlob, path, d),
352    }
353}
354
355/// Decodes exactly `N` bytes of lowercase hex.
356pub(crate) fn decode_hex_lower<const N: usize>(s: &str) -> Option<[u8; N]> {
357    let b = s.as_bytes();
358    if b.len() != 2 * N {
359        return None;
360    }
361    let mut out = [0u8; N];
362    for (i, o) in out.iter_mut().enumerate() {
363        let hi = hexval(b[2 * i])?;
364        let lo = hexval(b[2 * i + 1])?;
365        *o = (hi << 4) | lo;
366    }
367    Some(out)
368}
369
370fn hexval(c: u8) -> Option<u8> {
371    match c {
372        b'0'..=b'9' => Some(c - b'0'),
373        b'a'..=b'f' => Some(c - b'a' + 10),
374        _ => None,
375    }
376}
377
378/// Lowercase hex of bytes.
379pub fn to_hex(bytes: &[u8]) -> String {
380    const HEX: &[u8; 16] = b"0123456789abcdef";
381    let mut s = String::with_capacity(bytes.len() * 2);
382    for &b in bytes {
383        s.push(char::from(HEX[usize::from(b >> 4)]));
384        s.push(char::from(HEX[usize::from(b & 0xF)]));
385    }
386    s
387}
388
389/// Whether a code point is assigned in Unicode 15.1, by the spec's own table.
390pub fn is_assigned_15_1(c: char) -> bool {
391    let cp = c as u32;
392    tables::ASSIGNED
393        .binary_search_by(|&(lo, hi)| {
394            if hi < cp {
395                core::cmp::Ordering::Less
396            } else if lo > cp {
397                core::cmp::Ordering::Greater
398            } else {
399                core::cmp::Ordering::Equal
400            }
401        })
402        .is_ok()
403}
404
405/// The string rules of 3.5 in order: characters, NFC, then the byte limit.
406/// `allow_tab_lf` relaxes the control rule for function bodies (7).
407pub(crate) fn check_text(s: &str, limit: usize, allow_tab_lf: bool, path: &str) -> Result<()> {
408    for c in s.chars() {
409        let cp = c as u32;
410        if (cp < 0x20 && !(allow_tab_lf && (c == '\t' || c == '\n'))) || (0x7F..=0x9F).contains(&cp) {
411            return err(ErrorCode::CanonString, path, format!("control character U+{cp:04X}"));
412        }
413        if !is_assigned_15_1(c) {
414            return err(ErrorCode::CanonString, path, format!("U+{cp:04X} is not assigned in Unicode 15.1"));
415        }
416    }
417    if !unicode_normalization::is_nfc(s) {
418        return err(ErrorCode::CanonString, path, "not in Normalization Form C");
419    }
420    if s.len() > limit {
421        return err(ErrorCode::CanonLimit, path, format!("longer than {limit} UTF-8 bytes"));
422    }
423    Ok(())
424}
425
426/// A string value (3.5).
427pub fn check_string(s: &str, path: &str) -> Result<()> {
428    check_text(s, limits::STRING, false, path)
429}
430
431/// A licence (8.1): grammar (`E_CANON_LICENCE`), then length (`E_CANON_LIMIT`).
432pub fn check_licence(s: &str, path: &str) -> Result<()> {
433    if let Some(rest) = s.strip_prefix("https://") {
434        if rest.is_empty() || !rest.bytes().all(|c| (0x21..=0x7E).contains(&c)) {
435            return err(ErrorCode::CanonLicence, path, "a licence URL is https:// and printable ASCII");
436        }
437        if s.len() > limits::LICENCE_URL {
438            return err(ErrorCode::CanonLimit, path, "licence URL longer than 512 characters");
439        }
440        return Ok(());
441    }
442    let b = s.as_bytes();
443    let ok = !b.is_empty()
444        && b[0].is_ascii_alphanumeric()
445        && b.iter().all(|&c| c.is_ascii_alphanumeric() || c == b'.' || c == b'+' || c == b'-');
446    if !ok {
447        return err(ErrorCode::CanonLicence, path, format!("not an SPDX identifier or https URL: {s:?}"));
448    }
449    if s.len() > limits::LICENCE_SPDX {
450        return err(ErrorCode::CanonLimit, path, "licence identifier longer than 64 characters");
451    }
452    Ok(())
453}
454
455/// Whether `s` is a currency code of `units.md` 2.1 (withdrawn codes included).
456pub fn is_currency(s: &str) -> bool {
457    tables::CURRENCIES.contains(&s)
458}
459
460/// Whether `s` is an index unit (`units.md` 2.4).
461pub fn is_index_unit(s: &str) -> bool {
462    if s.len() > limits::INDEX_UNIT {
463        return false;
464    }
465    let Some(rest) = s.strip_prefix("index:") else { return false };
466    let Some((period, value)) = rest.split_once('=') else { return false };
467    let period_ok = match period.len() {
468        4 => is_date(&format!("{period}-01-01")),
469        7 => is_date(&format!("{period}-01")),
470        10 => is_date(period),
471        _ => false,
472    };
473    // Shapes like "2015-1-01" must not slip through the padding above.
474    let shape_ok =
475        period.bytes().enumerate().all(|(i, c)| if i == 4 || i == 7 { c == b'-' } else { c.is_ascii_digit() });
476    period_ok && shape_ok && is_number_grammar(value) && !value.starts_with('-') && value != "0"
477}
478
479/// Whether `s` is a literal unit: listed in `units.md` 2.1-2.3, or an index unit.
480pub fn is_literal_unit(s: &str) -> bool {
481    is_currency(s) || tables::PHYSICAL.contains(&s) || tables::RATIOS.contains(&s) || is_index_unit(s)
482}
483
484#[cfg(test)]
485mod tests {
486    use super::*;
487
488    #[test]
489    fn names() {
490        for ok in ["fx_rate", "base", "a01", "v2_final"] {
491            assert!(is_name_grammar(ok), "{ok}");
492        }
493        for bad in ["FxRate", "_x", "x_", "a__b", "1st", ""] {
494            assert!(!is_name_grammar(bad), "{bad}");
495        }
496    }
497
498    #[test]
499    fn numbers() {
500        for ok in ["0", "-1", "1.5", "0.001", "10"] {
501            assert!(is_number_grammar(ok), "{ok}");
502        }
503        for bad in ["-0", "01", "1.", ".5", "1.50", "+1", "1e3", "", "-", "0.0"] {
504            assert!(!is_number_grammar(bad), "{bad}");
505        }
506    }
507
508    #[test]
509    fn index_units() {
510        assert!(is_index_unit("index:2015=100"));
511        assert!(is_index_unit("index:1984-01-03=1000"));
512        assert!(is_index_unit("index:2016-02-29=0.5"));
513        for bad in [
514            "index:0000=1",
515            "index:2015=0",
516            "index:2015=-1",
517            "index:2015-13=1",
518            "index:2015-02-29=1",
519            "index:2015=1.0",
520            "index:15=1",
521        ] {
522            assert!(!is_index_unit(bad), "{bad}");
523        }
524    }
525
526    #[test]
527    fn instants() {
528        assert!(is_instant("2024-03-15T16:00:00.000000Z"));
529        assert!(!is_instant("2024-03-15T24:00:00.000000Z"));
530        assert!(!is_instant("2024-03-15T23:59:60.000000Z"));
531        assert!(!is_instant("2023-02-29T00:00:00.000000Z"));
532    }
533}
datumesh_core/lexical.rs

datumesh_core/
lexical.rs
