datumesh_core/
lexical.rs

1//! Lexical forms: names (2), scalars (3), hex, licences, key references (9.1) and units (7.1).
2//!
3//! Every check takes the path of the value it checks, for the error it may return. Each value
4//! is checked against its lexical form first and its length limit second (SPEC 4.5).
5
6use crate::error::{err, ErrorCode, Result};
7use crate::tables;
8
9/// Limits of SPEC 4.5.
10pub mod limits {
11    /// Characters in a function name, path segment, argument name or code language.
12    pub const NAME: usize = 64;
13    /// Characters in `<namespace>/<function>`.
14    pub const FUNCTION_NAME: usize = 512;
15    /// Arguments per function or cell.
16    pub const ARGS: usize = 32;
17    /// Characters in a number.
18    pub const NUMBER: usize = 128;
19    /// UTF-8 bytes in a string value.
20    pub const STRING: usize = 1024;
21    /// UTF-8 bytes in a receipt summary.
22    pub const SUMMARY: usize = 1024;
23    /// UTF-8 bytes in a function body.
24    pub const BODY: usize = 1_048_576;
25    /// Entries in a receipt's or descriptor's `inputs`.
26    pub const INPUTS: usize = 4096;
27    /// Characters in an SPDX licence identifier.
28    pub const LICENCE_SPDX: usize = 64;
29    /// Characters in a licence URL.
30    pub const LICENCE_URL: usize = 512;
31    /// Keys in one key publication.
32    pub const KEYS: usize = 64;
33    /// Characters in a blob media type (3.7).
34    pub const MEDIA_TYPE: usize = 128;
35    /// Characters in an index unit (`units.md` 2.4).
36    pub const INDEX_UNIT: usize = 64;
37    /// The largest blob length, 2^53 - 1.
38    pub const BLOB_LENGTH: u64 = 9_007_199_254_740_991;
39}
40
41fn is_lower_digit(c: u8) -> bool {
42    c.is_ascii_lowercase() || c.is_ascii_digit()
43}
44
45/// The grammar of 2.2 without its length limit.
46pub(crate) fn is_name_grammar(s: &str) -> bool {
47    let b = s.as_bytes();
48    match b.first() {
49        Some(c) if c.is_ascii_lowercase() => {}
50        _ => return false,
51    }
52    let mut prev_underscore = false;
53    for &c in &b[1..] {
54        if c == b'_' {
55            if prev_underscore {
56                return false;
57            }
58            prev_underscore = true;
59        } else if is_lower_digit(c) {
60            prev_underscore = false;
61        } else {
62            return false;
63        }
64    }
65    !prev_underscore
66}
67
68/// A name (2.2): function names, path segments and code languages. `E_CANON_NAME`, then `E_CANON_LIMIT`.
69pub fn check_name(s: &str, path: &str) -> Result<()> {
70    if !is_name_grammar(s) {
71        return err(ErrorCode::CanonName, path, format!("not a lowercase snake_case name: {s:?}"));
72    }
73    if s.len() > limits::NAME {
74        return err(ErrorCode::CanonLimit, path, "name longer than 64 characters");
75    }
76    Ok(())
77}
78
79/// An argument name (2.4): grammar (`E_CANON_ARG_NAME`), length (`E_CANON_LIMIT`), then reserved names.
80pub fn check_arg_name(s: &str, path: &str) -> Result<()> {
81    if !is_name_grammar(s) {
82        return err(ErrorCode::CanonArgName, path, format!("not a lowercase snake_case name: {s:?}"));
83    }
84    if s.len() > limits::NAME {
85        return err(ErrorCode::CanonLimit, path, "name longer than 64 characters");
86    }
87    if s == "asof" || s.starts_with("dm_") {
88        return err(ErrorCode::CanonArgName, path, format!("reserved argument name {s:?}"));
89    }
90    Ok(())
91}
92
93/// A DNS name (2.1). Every failure is `E_CANON_NAME`.
94pub fn check_dns_name(s: &str, path: &str) -> Result<()> {
95    if s.len() > 253 {
96        return err(ErrorCode::CanonName, path, "DNS name longer than 253 characters");
97    }
98    let labels: Vec<&str> = s.split('.').collect();
99    if labels.len() < 2 {
100        return err(ErrorCode::CanonName, path, format!("DNS name needs at least two labels: {s:?}"));
101    }
102    for l in &labels {
103        let b = l.as_bytes();
104        let ok = !b.is_empty()
105            && b.len() <= 63
106            && b.iter().all(|&c| is_lower_digit(c) || c == b'-')
107            && b[0] != b'-'
108            && b[b.len() - 1] != b'-';
109        if !ok {
110            return err(ErrorCode::CanonName, path, format!("bad DNS label {l:?}"));
111        }
112    }
113    if labels.last().is_some_and(|l| l.bytes().all(|c| c.is_ascii_digit())) {
114        return err(ErrorCode::CanonName, path, "top-level label is numeric");
115    }
116    Ok(())
117}
118
119/// A namespace (2.3): a DNS name, then path segments that are names.
120pub fn check_namespace(s: &str, path: &str) -> Result<()> {
121    let mut parts = s.split('/');
122    check_dns_name(parts.next().unwrap_or(""), path)?;
123    for seg in parts {
124        check_name(seg, path)?;
125    }
126    Ok(())
127}
128
129/// The host of a namespace: its DNS name.
130pub(crate) fn host_of(namespace: &str) -> &str {
131    namespace.split('/').next().unwrap_or(namespace)
132}
133
134/// A fully qualified function name (2.3): namespace, `/`, name; at most 512 characters.
135pub fn check_function_name(s: &str, path: &str) -> Result<()> {
136    let Some((ns, f)) = s.rsplit_once('/') else {
137        return err(ErrorCode::CanonName, path, format!("function name has no namespace: {s:?}"));
138    };
139    check_namespace(ns, path)?;
140    check_name(f, path)?;
141    if s.len() > limits::FUNCTION_NAME {
142        return err(ErrorCode::CanonLimit, path, "function name longer than 512 characters");
143    }
144    Ok(())
145}
146
147/// Lowercase hex of exactly `digits` digits (`E_CANON_HEX`).
148pub fn check_hex(s: &str, digits: usize, path: &str) -> Result<()> {
149    if s.len() != digits || !s.bytes().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) {
150        return err(ErrorCode::CanonHex, path, format!("expected {digits} lowercase hex digits"));
151    }
152    Ok(())
153}
154
155/// A key reference `<host>#<key id>` (9.1).
156pub fn check_key_ref(s: &str, path: &str) -> Result<()> {
157    let Some((host, id)) = s.split_once('#') else {
158        return err(ErrorCode::CanonName, path, "a key reference is <host>#<key id>");
159    };
160    check_dns_name(host, path)?;
161    check_hex(id, 16, path)
162}
163
164/// The number grammar of 3.1 without the length limit.
165pub(crate) fn is_number_grammar(s: &str) -> bool {
166    let b = s.as_bytes();
167    let mut i = 0;
168    if b.first() == Some(&b'-') {
169        i = 1;
170    }
171    match b.get(i) {
172        Some(b'0') => i += 1,
173        Some(b'1'..=b'9') => {
174            while b.get(i).is_some_and(u8::is_ascii_digit) {
175                i += 1;
176            }
177        }
178        _ => return false,
179    }
180    if b.get(i) == Some(&b'.') {
181        let frac = &b[i + 1..];
182        if frac.is_empty() || !frac.iter().all(u8::is_ascii_digit) || frac[frac.len() - 1] == b'0' {
183            return false;
184        }
185        i = b.len();
186    }
187    i == b.len() && s != "-0"
188}
189
190/// A number (3.1): `E_CANON_NUMBER`, then `E_CANON_LIMIT`.
191pub fn check_number(s: &str, path: &str) -> Result<()> {
192    if !is_number_grammar(s) {
193        return err(ErrorCode::CanonNumber, path, format!("not a canonical decimal: {s:?}"));
194    }
195    if s.len() > limits::NUMBER {
196        return err(ErrorCode::CanonLimit, path, "number longer than 128 characters");
197    }
198    Ok(())
199}
200
201fn digits(b: &[u8]) -> Option<u32> {
202    if b.is_empty() || !b.iter().all(u8::is_ascii_digit) {
203        return None;
204    }
205    Some(b.iter().fold(0u32, |a, &c| a * 10 + u32::from(c - b'0')))
206}
207
208fn days_in_month(y: u32, m: u32) -> u32 {
209    match m {
210        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
211        4 | 6 | 9 | 11 => 30,
212        2 if (y.is_multiple_of(4) && !y.is_multiple_of(100)) || y.is_multiple_of(400) => 29,
213        2 => 28,
214        _ => 0,
215    }
216}
217
218/// `YYYY-MM-DD`, a real day in years 0001-9999.
219pub(crate) fn is_date(s: &str) -> bool {
220    let b = s.as_bytes();
221    if b.len() != 10 || b[4] != b'-' || b[7] != b'-' {
222        return false;
223    }
224    let (Some(y), Some(m), Some(d)) = (digits(&b[0..4]), digits(&b[5..7]), digits(&b[8..10])) else {
225        return false;
226    };
227    (1..=9999).contains(&y) && (1..=12).contains(&m) && d >= 1 && d <= days_in_month(y, m)
228}
229
230/// A date (3.2), `E_CANON_DATE`.
231pub fn check_date(s: &str, path: &str) -> Result<()> {
232    if !is_date(s) {
233        return err(ErrorCode::CanonDate, path, format!("not a YYYY-MM-DD calendar date: {s:?}"));
234    }
235    Ok(())
236}
237
238/// `YYYY-MM-DDTHH:MM:SS.ffffffZ`.
239pub(crate) fn is_instant(s: &str) -> bool {
240    let b = s.as_bytes();
241    if b.len() != 27 || !s.is_char_boundary(10) || !is_date(&s[..10]) {
242        return false;
243    }
244    if b[10] != b'T' || b[13] != b':' || b[16] != b':' || b[19] != b'.' || b[26] != b'Z' {
245        return false;
246    }
247    let (Some(h), Some(mi), Some(se), Some(_)) =
248        (digits(&b[11..13]), digits(&b[14..16]), digits(&b[17..19]), digits(&b[20..26]))
249    else {
250        return false;
251    };
252    h < 24 && mi < 60 && se < 60
253}
254
255/// An instant (3.3), `E_CANON_INSTANT`.
256pub fn check_instant(s: &str, path: &str) -> Result<()> {
257    if !is_instant(s) {
258        return err(ErrorCode::CanonInstant, path, format!("not YYYY-MM-DDTHH:MM:SS.ffffffZ: {s:?}"));
259    }
260    Ok(())
261}
262
263fn is_restricted_name(s: &str) -> bool {
264    let b = s.as_bytes();
265    !b.is_empty() && is_lower_digit(b[0]) && b.iter().all(|&c| is_lower_digit(c) || b"!#$&-^_.+".contains(&c))
266}
267
268/// A blob (3.7): its sha256, length and media type. Every failure is `E_CANON_BLOB`.
269pub(crate) fn parse_blob(s: &str) -> core::result::Result<([u8; 32], u64, &str), String> {
270    let mut parts = s.split(':');
271    let (Some(h), Some(len), Some(mt), None) = (parts.next(), parts.next(), parts.next(), parts.next()) else {
272        return Err("a blob is <sha256>:<length>:<media type>".into());
273    };
274    let hash = match decode_hex_lower::<32>(h) {
275        Some(x) => x,
276        None => return Err("the hash is 64 lowercase hex digits".into()),
277    };
278    let lb = len.as_bytes();
279    if lb.is_empty() || !lb.iter().all(u8::is_ascii_digit) || (lb.len() > 1 && lb[0] == b'0') || lb.len() > 16 {
280        return Err("the length is a canonical non-negative integer".into());
281    }
282    let n: u64 = len.parse().map_err(|_| String::from("bad length"))?;
283    if n > limits::BLOB_LENGTH {
284        return Err("length above 9007199254740991".into());
285    }
286    let ok_mt = match mt.split_once('/') {
287        Some((t, st)) => is_restricted_name(t) && is_restricted_name(st),
288        None => false,
289    };
290    if !ok_mt {
291        return Err(format!("not a lowercase type/subtype media type: {mt:?}"));
292    }
293    if mt.len() > limits::MEDIA_TYPE {
294        return Err("media type longer than 128 characters".into());
295    }
296    Ok((hash, n, mt))
297}
298
299/// A blob (3.7), `E_CANON_BLOB`.
300pub fn check_blob(s: &str, path: &str) -> Result<()> {
301    match parse_blob(s) {
302        Ok(_) => Ok(()),
303        Err(d) => err(ErrorCode::CanonBlob, path, d),
304    }
305}
306
307/// Decodes exactly `N` bytes of lowercase hex.
308pub(crate) fn decode_hex_lower<const N: usize>(s: &str) -> Option<[u8; N]> {
309    let b = s.as_bytes();
310    if b.len() != 2 * N {
311        return None;
312    }
313    let mut out = [0u8; N];
314    for (i, o) in out.iter_mut().enumerate() {
315        let hi = hexval(b[2 * i])?;
316        let lo = hexval(b[2 * i + 1])?;
317        *o = (hi << 4) | lo;
318    }
319    Some(out)
320}
321
322fn hexval(c: u8) -> Option<u8> {
323    match c {
324        b'0'..=b'9' => Some(c - b'0'),
325        b'a'..=b'f' => Some(c - b'a' + 10),
326        _ => None,
327    }
328}
329
330/// Lowercase hex of bytes.
331pub fn to_hex(bytes: &[u8]) -> String {
332    const HEX: &[u8; 16] = b"0123456789abcdef";
333    let mut s = String::with_capacity(bytes.len() * 2);
334    for &b in bytes {
335        s.push(char::from(HEX[usize::from(b >> 4)]));
336        s.push(char::from(HEX[usize::from(b & 0xF)]));
337    }
338    s
339}
340
341/// Whether a code point is assigned in Unicode 15.1, by the spec's own table.
342pub fn is_assigned_15_1(c: char) -> bool {
343    let cp = c as u32;
344    tables::ASSIGNED
345        .binary_search_by(|&(lo, hi)| {
346            if hi < cp {
347                core::cmp::Ordering::Less
348            } else if lo > cp {
349                core::cmp::Ordering::Greater
350            } else {
351                core::cmp::Ordering::Equal
352            }
353        })
354        .is_ok()
355}
356
357/// The string rules of 3.5 in order: characters, NFC, then the byte limit.
358/// `allow_tab_lf` relaxes the control rule for function bodies (7).
359pub(crate) fn check_text(s: &str, limit: usize, allow_tab_lf: bool, path: &str) -> Result<()> {
360    for c in s.chars() {
361        let cp = c as u32;
362        if (cp < 0x20 && !(allow_tab_lf && (c == '\t' || c == '\n'))) || (0x7F..=0x9F).contains(&cp) {
363            return err(ErrorCode::CanonString, path, format!("control character U+{cp:04X}"));
364        }
365        if !is_assigned_15_1(c) {
366            return err(ErrorCode::CanonString, path, format!("U+{cp:04X} is not assigned in Unicode 15.1"));
367        }
368    }
369    if !unicode_normalization::is_nfc(s) {
370        return err(ErrorCode::CanonString, path, "not in Normalization Form C");
371    }
372    if s.len() > limit {
373        return err(ErrorCode::CanonLimit, path, format!("longer than {limit} UTF-8 bytes"));
374    }
375    Ok(())
376}
377
378/// A string value (3.5).
379pub fn check_string(s: &str, path: &str) -> Result<()> {
380    check_text(s, limits::STRING, false, path)
381}
382
383/// A licence (8.1): grammar (`E_CANON_LICENCE`), then length (`E_CANON_LIMIT`).
384pub fn check_licence(s: &str, path: &str) -> Result<()> {
385    if let Some(rest) = s.strip_prefix("https://") {
386        if rest.is_empty() || !rest.bytes().all(|c| (0x21..=0x7E).contains(&c)) {
387            return err(ErrorCode::CanonLicence, path, "a licence URL is https:// and printable ASCII");
388        }
389        if s.len() > limits::LICENCE_URL {
390            return err(ErrorCode::CanonLimit, path, "licence URL longer than 512 characters");
391        }
392        return Ok(());
393    }
394    let b = s.as_bytes();
395    let ok = !b.is_empty()
396        && b[0].is_ascii_alphanumeric()
397        && b.iter().all(|&c| c.is_ascii_alphanumeric() || c == b'.' || c == b'+' || c == b'-');
398    if !ok {
399        return err(ErrorCode::CanonLicence, path, format!("not an SPDX identifier or https URL: {s:?}"));
400    }
401    if s.len() > limits::LICENCE_SPDX {
402        return err(ErrorCode::CanonLimit, path, "licence identifier longer than 64 characters");
403    }
404    Ok(())
405}
406
407/// Whether `s` is a currency code of `units.md` 2.1 (withdrawn codes included).
408pub fn is_currency(s: &str) -> bool {
409    tables::CURRENCIES.contains(&s)
410}
411
412/// Whether `s` is an index unit (`units.md` 2.4).
413pub fn is_index_unit(s: &str) -> bool {
414    if s.len() > limits::INDEX_UNIT {
415        return false;
416    }
417    let Some(rest) = s.strip_prefix("index:") else { return false };
418    let Some((period, value)) = rest.split_once('=') else { return false };
419    let period_ok = match period.len() {
420        4 => is_date(&format!("{period}-01-01")),
421        7 => is_date(&format!("{period}-01")),
422        10 => is_date(period),
423        _ => false,
424    };
425    // Shapes like "2015-1-01" must not slip through the padding above.
426    let shape_ok =
427        period.bytes().enumerate().all(|(i, c)| if i == 4 || i == 7 { c == b'-' } else { c.is_ascii_digit() });
428    period_ok && shape_ok && is_number_grammar(value) && !value.starts_with('-') && value != "0"
429}
430
431/// Whether `s` is a literal unit: listed in `units.md` 2.1-2.3, or an index unit.
432pub fn is_literal_unit(s: &str) -> bool {
433    is_currency(s) || tables::PHYSICAL.contains(&s) || tables::RATIOS.contains(&s) || is_index_unit(s)
434}
435
436#[cfg(test)]
437mod tests {
438    use super::*;
439
440    #[test]
441    fn names() {
442        for ok in ["fx_rate", "base", "a01", "v2_final"] {
443            assert!(is_name_grammar(ok), "{ok}");
444        }
445        for bad in ["FxRate", "_x", "x_", "a__b", "1st", ""] {
446            assert!(!is_name_grammar(bad), "{bad}");
447        }
448    }
449
450    #[test]
451    fn numbers() {
452        for ok in ["0", "-1", "1.5", "0.001", "10"] {
453            assert!(is_number_grammar(ok), "{ok}");
454        }
455        for bad in ["-0", "01", "1.", ".5", "1.50", "+1", "1e3", "", "-", "0.0"] {
456            assert!(!is_number_grammar(bad), "{bad}");
457        }
458    }
459
460    #[test]
461    fn index_units() {
462        assert!(is_index_unit("index:2015=100"));
463        assert!(is_index_unit("index:1984-01-03=1000"));
464        assert!(is_index_unit("index:2016-02-29=0.5"));
465        for bad in [
466            "index:0000=1",
467            "index:2015=0",
468            "index:2015=-1",
469            "index:2015-13=1",
470            "index:2015-02-29=1",
471            "index:2015=1.0",
472            "index:15=1",
473        ] {
474            assert!(!is_index_unit(bad), "{bad}");
475        }
476    }
477
478    #[test]
479    fn instants() {
480        assert!(is_instant("2024-03-15T16:00:00.000000Z"));
481        assert!(!is_instant("2024-03-15T24:00:00.000000Z"));
482        assert!(!is_instant("2024-03-15T23:59:60.000000Z"));
483        assert!(!is_instant("2023-02-29T00:00:00.000000Z"));
484    }
485}
datumesh_core/lexical.rs

datumesh_core/
lexical.rs
