use std::fmt;
use sha2::{Digest, Sha256};
#[derive(Debug, Clone, PartialEq)]
pub enum Json {
Null,
Bool(bool),
Number(f64),
Text(String),
Array(Vec<Self>),
Object(Vec<(String, Self)>),
}
impl Json {
#[must_use]
pub fn get(&self, key: &str) -> Option<&Self> {
match self {
Self::Object(kv) => kv.iter().rev().find(|(k, _)| k == key).map(|(_, v)| v),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JsonError {
pub at: usize,
pub reason: &'static str,
}
impl fmt::Display for JsonError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "JSON: {} at byte {}", self.reason, self.at)
}
}
impl std::error::Error for JsonError {}
pub fn parse_json(text: &str) -> Result<Json, JsonError> {
let mut p = JsonParser {
s: text.as_bytes(),
i: 0,
depth: 0,
};
p.ws();
let v = p.value()?;
p.ws();
if p.i == p.s.len() {
Ok(v)
} else {
Err(p.err("text after the value"))
}
}
const MAX_DEPTH: usize = 512;
struct JsonParser<'a> {
s: &'a [u8],
i: usize,
depth: usize,
}
impl JsonParser<'_> {
const fn err(&self, reason: &'static str) -> JsonError {
JsonError { at: self.i, reason }
}
fn ws(&mut self) {
while matches!(self.s.get(self.i), Some(b' ' | b'\t' | b'\n' | b'\r')) {
self.i += 1;
}
}
fn lit(&mut self, word: &[u8], v: Json) -> Result<Json, JsonError> {
if self.s[self.i..].starts_with(word) {
self.i += word.len();
Ok(v)
} else {
Err(self.err("unknown literal"))
}
}
fn value(&mut self) -> Result<Json, JsonError> {
match self.s.get(self.i) {
None => Err(self.err("unexpected end")),
Some(b'n') => self.lit(b"null", Json::Null),
Some(b't') => self.lit(b"true", Json::Bool(true)),
Some(b'f') => self.lit(b"false", Json::Bool(false)),
Some(b'"') => self.string().map(Json::Text),
Some(b'[') => self.nested(Self::array),
Some(b'{') => self.nested(Self::object),
Some(b'-' | b'0'..=b'9') => self.number(),
Some(_) => Err(self.err("unexpected character")),
}
}
fn nested(&mut self, f: fn(&mut Self) -> Result<Json, JsonError>) -> Result<Json, JsonError> {
if self.depth >= MAX_DEPTH {
return Err(self.err("nested too deeply"));
}
self.depth += 1;
let v = f(self);
self.depth -= 1;
v
}
fn array(&mut self) -> Result<Json, JsonError> {
self.i += 1;
let mut out = Vec::new();
self.ws();
if self.s.get(self.i) == Some(&b']') {
self.i += 1;
return Ok(Json::Array(out));
}
loop {
self.ws();
out.push(self.value()?);
self.ws();
match self.s.get(self.i) {
Some(b',') => self.i += 1,
Some(b']') => {
self.i += 1;
return Ok(Json::Array(out));
}
_ => return Err(self.err("`,` or `]` expected")),
}
}
}
fn object(&mut self) -> Result<Json, JsonError> {
self.i += 1;
let mut out = Vec::new();
self.ws();
if self.s.get(self.i) == Some(&b'}') {
self.i += 1;
return Ok(Json::Object(out));
}
loop {
self.ws();
if self.s.get(self.i) != Some(&b'"') {
return Err(self.err("key expected"));
}
let k = self.string()?;
self.ws();
if self.s.get(self.i) != Some(&b':') {
return Err(self.err("`:` expected"));
}
self.i += 1;
self.ws();
out.push((k, self.value()?));
self.ws();
match self.s.get(self.i) {
Some(b',') => self.i += 1,
Some(b'}') => {
self.i += 1;
return Ok(Json::Object(out));
}
_ => return Err(self.err("`,` or `}` expected")),
}
}
}
fn hex4(&mut self) -> Result<u32, JsonError> {
let h = self
.s
.get(self.i..self.i + 4)
.ok_or_else(|| self.err("short \\u escape"))?;
let text = std::str::from_utf8(h).map_err(|_| self.err("bad \\u escape"))?;
let v = u32::from_str_radix(text, 16).map_err(|_| self.err("bad \\u escape"))?;
self.i += 4;
Ok(v)
}
fn string(&mut self) -> Result<String, JsonError> {
self.i += 1;
let mut out = String::new();
loop {
let start = self.i;
while let Some(&c) = self.s.get(self.i) {
if c == b'"' || c == b'\\' || c < 0x20 {
break;
}
self.i += 1;
}
out.push_str(
std::str::from_utf8(&self.s[start..self.i]).map_err(|_| JsonError {
at: start,
reason: "not UTF-8",
})?,
);
match self.s.get(self.i) {
None => return Err(self.err("unterminated string")),
Some(b'"') => {
self.i += 1;
return Ok(out);
}
Some(b'\\') => {
self.i += 1;
let e = *self
.s
.get(self.i)
.ok_or_else(|| self.err("unterminated string"))?;
self.i += 1;
match e {
b'"' => out.push('"'),
b'\\' => out.push('\\'),
b'/' => out.push('/'),
b'b' => out.push('\u{8}'),
b'f' => out.push('\u{c}'),
b'n' => out.push('\n'),
b'r' => out.push('\r'),
b't' => out.push('\t'),
b'u' => {
let hi = self.hex4()?;
let cp = if (0xD800..0xDC00).contains(&hi) {
if !self.s[self.i..].starts_with(b"\\u") {
return Err(self.err("lone surrogate"));
}
self.i += 2;
let lo = self.hex4()?;
if !(0xDC00..0xE000).contains(&lo) {
return Err(self.err("lone surrogate"));
}
0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00)
} else {
hi
};
out.push(char::from_u32(cp).ok_or_else(|| self.err("lone surrogate"))?);
}
_ => return Err(self.err("unknown escape")),
}
}
Some(_) => return Err(self.err("control character in string")),
}
}
}
fn digits(&mut self) -> usize {
let start = self.i;
while matches!(self.s.get(self.i), Some(b'0'..=b'9')) {
self.i += 1;
}
self.i - start
}
fn number(&mut self) -> Result<Json, JsonError> {
let start = self.i;
if self.s.get(self.i) == Some(&b'-') {
self.i += 1;
}
match self.s.get(self.i) {
Some(b'0') => self.i += 1,
Some(b'1'..=b'9') => {
self.digits();
}
_ => return Err(self.err("digit expected")),
}
if self.s.get(self.i) == Some(&b'.') {
self.i += 1;
if self.digits() == 0 {
return Err(self.err("digit expected after `.`"));
}
}
if matches!(self.s.get(self.i), Some(b'e' | b'E')) {
self.i += 1;
if matches!(self.s.get(self.i), Some(b'+' | b'-')) {
self.i += 1;
}
if self.digits() == 0 {
return Err(self.err("digit expected in exponent"));
}
}
let text =
std::str::from_utf8(&self.s[start..self.i]).map_err(|_| self.err("bad number"))?;
text.parse::<f64>()
.map(Json::Number)
.map_err(|_| self.err("bad number"))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum InputType {
Number,
Integer,
Text,
List(Box<Self>),
Record(Vec<Field>),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Field {
pub name: String,
pub optional: bool,
pub ty: InputType,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TypeError(pub String);
impl fmt::Display for TypeError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "x-input type: {}", self.0)
}
}
impl std::error::Error for TypeError {}
impl InputType {
pub fn parse(text: &str) -> Result<Self, TypeError> {
let (t, rest) = Self::parse_prefix(text.trim())?;
if rest.trim().is_empty() {
Ok(t)
} else {
Err(TypeError(format!("text after the type: `{}`", rest.trim())))
}
}
fn parse_prefix(s: &str) -> Result<(Self, &str), TypeError> {
let s = s.trim_start();
for (word, t) in [
("number", Self::Number),
("integer", Self::Integer),
("text", Self::Text),
] {
if let Some(rest) = s.strip_prefix(word) {
if !rest.starts_with(|c: char| c.is_alphanumeric() || c == '_') {
return Ok((t, rest));
}
}
}
if let Some(rest) = s.strip_prefix("list of ") {
let (inner, rest) = Self::parse_prefix(rest)?;
return Ok((Self::List(Box::new(inner)), rest));
}
if let Some(mut rest) = s.strip_prefix("record(") {
let mut fields: Vec<Field> = Vec::new();
loop {
let (name, after) = rest
.split_once(':')
.ok_or_else(|| TypeError(format!("field name expected at `{rest}`")))?;
let name = name.trim();
let ok_name = !name.is_empty()
&& name
.chars()
.all(|c| c.is_alphanumeric() || c == '_' || c == '-')
&& !name.starts_with(|c: char| c.is_ascii_digit());
if !ok_name {
return Err(TypeError(format!("bad field name `{name}`")));
}
if fields.iter().any(|f| f.name == name) {
return Err(TypeError(format!("field `{name}` appears twice")));
}
let after = after.trim_start();
let (optional, after) = after
.strip_prefix("optional ")
.map_or((false, after), |r| (true, r));
let (ty, after) = Self::parse_prefix(after)?;
fields.push(Field {
name: name.to_owned(),
optional,
ty,
});
let after = after.trim_start();
if let Some(r) = after.strip_prefix(',') {
rest = r;
} else if let Some(r) = after.strip_prefix(')') {
return Ok((Self::Record(fields), r));
} else {
return Err(TypeError(format!("`,` or `)` expected at `{after}`")));
}
}
}
Err(TypeError(format!("type expected at `{s}`")))
}
#[must_use]
pub fn canonical(&self) -> String {
match self {
Self::Number => "number".to_owned(),
Self::Integer => "integer".to_owned(),
Self::Text => "text".to_owned(),
Self::List(t) => format!("list of {}", t.canonical()),
Self::Record(fs) => {
let parts: Vec<String> = fs
.iter()
.map(|f| {
format!(
"{}: {}{}",
f.name,
if f.optional { "optional " } else { "" },
f.ty.canonical()
)
})
.collect();
format!("record({})", parts.join(", "))
}
}
}
#[must_use]
pub fn matches(&self, v: &Json) -> bool {
match (self, v) {
(Self::Number, Json::Number(x)) => x.is_finite(),
(Self::Integer, Json::Number(x)) => x.is_finite() && x.fract() == 0.0,
(Self::Text, Json::Text(_)) => true,
(Self::List(t), Json::Array(xs)) => xs.iter().all(|x| t.matches(x)),
(Self::Record(fs), Json::Object(_)) => fs
.iter()
.all(|f| v.get(&f.name).map_or(f.optional, |x| f.ty.matches(x))),
_ => false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LawKind {
Audit,
Quantitative,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Read<'a> {
Value(&'a Json),
NotMeasured,
Rejected(String),
}
#[must_use]
pub fn read<'a>(name: &str, ty: &InputType, value: Option<&'a Json>, kind: LawKind) -> Read<'a> {
match value {
None => Read::NotMeasured,
Some(v) if ty.matches(v) => Read::Value(v),
Some(_) => match kind {
LawKind::Audit => Read::NotMeasured,
LawKind::Quantitative => {
Read::Rejected(format!("`{name}` does not match `{}`", ty.canonical()))
}
},
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MetricExpr {
Count {
input: String,
},
Distinct {
input: String,
field: String,
as_set: bool,
},
}
impl MetricExpr {
pub fn parse(text: &str) -> Result<Self, TypeError> {
let t: String = text.chars().filter(|c| !c.is_whitespace()).collect();
let bad = || TypeError(format!("metric expression expected, got `{}`", text.trim()));
let name_ok = |n: &str| {
!n.is_empty()
&& n.chars()
.all(|c| c.is_alphanumeric() || c == '_' || c == '-')
&& !n.starts_with(|c: char| c.is_ascii_digit())
};
let path = |p: &str| -> Option<(String, String)> {
let (input, field) = p.split_once("[].")?;
(name_ok(input) && name_ok(field)).then(|| (input.to_owned(), field.to_owned()))
};
if let Some(inner) = t.strip_prefix("count(").and_then(|r| r.strip_suffix(')')) {
return if name_ok(inner) {
Ok(Self::Count {
input: inner.to_owned(),
})
} else {
Err(bad())
};
}
let inner = t
.strip_prefix("distinct(")
.and_then(|r| r.strip_suffix(')'))
.ok_or_else(bad)?;
let (inner, as_set) = inner
.strip_prefix("set(")
.and_then(|r| r.strip_suffix(')'))
.map_or((inner, false), |r| (r, true));
let (input, field) = path(inner).ok_or_else(bad)?;
Ok(Self::Distinct {
input,
field,
as_set,
})
}
#[must_use]
pub fn canonical(&self) -> String {
match self {
Self::Count { input } => format!("count({input})"),
Self::Distinct {
input,
field,
as_set: false,
} => format!("distinct({input}[].{field})"),
Self::Distinct {
input,
field,
as_set: true,
} => format!("distinct(set({input}[].{field}))"),
}
}
#[must_use]
pub fn input(&self) -> &str {
match self {
Self::Count { input } | Self::Distinct { input, .. } => input,
}
}
#[must_use]
pub fn evaluate(&self, value: Option<&Json>) -> Option<f64> {
let Some(Json::Array(items)) = value else {
return None;
};
#[allow(clippy::cast_precision_loss, reason = "counts are small")]
let count = |n: usize| n as f64;
match self {
Self::Count { .. } => Some(count(items.len())),
Self::Distinct { field, as_set, .. } => {
let mut seen: Vec<Vec<u8>> = Vec::new();
for it in items {
let v = it.get(field)?;
let key = if *as_set {
let Json::Array(xs) = v else { return None };
let mut ks: Vec<Vec<u8>> = xs.iter().map(value_key).collect();
ks.sort();
ks.dedup();
ks.concat_with_len()
} else {
value_key(v)
};
if !seen.contains(&key) {
seen.push(key);
}
}
Some(count(seen.len()))
}
}
}
}
fn value_key(v: &Json) -> Vec<u8> {
let mut out = Vec::new();
match v {
Json::Null => out.push(0),
Json::Bool(b) => out.extend([1, u8::from(*b)]),
Json::Number(x) => {
out.push(2);
out.extend((x + 0.0).to_bits().to_le_bytes());
}
Json::Text(t) => {
out.push(3);
out.extend(t.as_bytes());
}
Json::Array(xs) => {
out.push(4);
out.extend(
xs.iter()
.map(value_key)
.collect::<Vec<_>>()
.concat_with_len(),
);
}
Json::Object(kv) => {
out.push(5);
for (k, x) in kv {
out.extend((k.len() as u64).to_le_bytes());
out.extend(k.as_bytes());
out.extend(value_key(x));
}
}
}
out
}
trait ConcatWithLen {
fn concat_with_len(&self) -> Vec<u8>;
}
impl ConcatWithLen for Vec<Vec<u8>> {
fn concat_with_len(&self) -> Vec<u8> {
let mut out = Vec::new();
for k in self {
out.extend((k.len() as u64).to_le_bytes());
out.extend(k);
}
out
}
}
#[must_use]
pub fn measurements(
law: &crate::audit_law::AuditLaw,
inputs: &Json,
) -> crate::audit_law::Measurements {
use crate::audit_law::{Clause, Measurements};
let mut m = Measurements::new();
let typed = |name: &str| law.inputs().iter().find(|(n, _)| n == name).map(|(_, t)| t);
let read_typed = |name: &str| -> Option<&Json> {
let ty = typed(name)?;
match read(name, ty, inputs.get(name), LawKind::Audit) {
Read::Value(v) => Some(v),
_ => None,
}
};
if let Json::Object(kv) = inputs {
let mut keys: Vec<&str> = kv.iter().map(|(k, _)| k.as_str()).collect();
keys.sort_unstable();
keys.dedup();
for k in keys {
if typed(k).is_some() || law.metrics().iter().any(|(n, _)| n == k) {
continue;
}
if let Some(Json::Number(x)) = inputs.get(k) {
if x.is_finite() {
m = m.with_number(k, *x);
}
}
}
}
for c in law.clauses() {
if let Clause::Range { key, .. } = c {
if let Some(Json::Array(xs)) = read_typed(key) {
let texts: Vec<&str> = xs
.iter()
.filter_map(|x| {
if let Json::Text(t) = x {
Some(t.as_str())
} else {
None
}
})
.collect();
m = m.with_range(key, &texts);
}
}
}
for (name, expr) in law.metrics() {
if let Some(v) = expr.evaluate(read_typed(expr.input())) {
m = m.with_number(name, v);
}
}
for (name, n) in law.at_least() {
if !m.number(name).is_some_and(|v| v >= *n) {
m = m.with_number(name, 0.0);
}
}
m
}
const DERIVATION_CASES: &[(&str, &str)] = &[
(
DERIVATION_LAW,
r#"{"b":[{"f":["x"],"id":"a"},{"f":["y"],"id":"a"}]}"#,
),
(
DERIVATION_LAW,
r#"{"b":[{"f":["x","x"],"id":"a"},{"f":["x"],"id":"b"}]}"#,
),
(
DERIVATION_LAW,
r#"{"b":[{"f":["y","x"],"id":"a"},{"f":["x","y"],"id":"a"}]}"#,
),
(
DERIVATION_LAW,
r#"{"b":[{"f":["1"],"id":"1"},{"f":["01"],"id":"1.0"}]}"#,
),
(DERIVATION_LAW, r#"{"b":[{"f":[]},{"f":["x"],"id":"a"}]}"#),
(
DERIVATION_LAW,
r#"{"b":[{"f":[],"id":"a"},{"f":[],"id":"a"}]}"#,
),
(DERIVATION_LAW, r#"{"b":[]}"#),
(DERIVATION_LAW, r#"{"b":[{"f":["x"],"id":5}]}"#),
(DERIVATION_LAW, r#"{"b":"x"}"#),
(DERIVATION_LAW, "{}"),
(
DERIVATION_LAW,
r#"{"b":[{"f":["x"],"id":"a"}],"sets":7,"ids":3}"#,
),
(DERIVATION_LAW, r#"{"count":5,"ids":3}"#),
(DERIVATION_LAW, r#"{"b":[],"n":4,"k":["q","p","q"]}"#),
];
const DERIVATION_LAW: &str = "\
x-input b list of record(f: list of text, id: optional text)
x-input k list of text
x-metric count = count(b)
x-metric sets = distinct(set(b[].f))
x-metric ids = distinct(b[].id)
x-at-least sets 2
begin audit
audit derivation
evidence count
evidence n
range k p q
expect sets == 2
expect ids == 1
end audit
";
#[must_use]
pub fn derivation_fingerprint() -> [u8; 32] {
use crate::audit_law::Clause;
let mut h = Sha256::new();
h.update(b"lol.audit-derivation");
for (law_text, inputs) in DERIVATION_CASES {
let law = audit_law_from_file(law_text).expect("fingerprint law parses");
let req = parse_json(inputs).expect("fingerprint inputs parse");
let m = measurements(&law, &req);
h.update((inputs.len() as u64).to_le_bytes());
h.update(inputs.as_bytes());
let mut names: Vec<&str> = law.metrics().iter().map(|(n, _)| n.as_str()).collect();
for c in law.clauses() {
match c {
Clause::Evidence { metric } | Clause::Expect { metric, .. } => names.push(metric),
Clause::Range { key, .. } => names.push(key),
}
}
names.sort_unstable();
names.dedup();
for name in names {
h.update((name.len() as u64).to_le_bytes());
h.update(name.as_bytes());
match m.number(name) {
Some(v) => {
h.update([1]);
h.update(v.to_bits().to_le_bytes());
}
None => h.update([0]),
}
match m.range(name) {
Some(r) => {
h.update([1]);
h.update((r.len() as u64).to_le_bytes());
for x in r {
h.update((x.len() as u64).to_le_bytes());
h.update(x.as_bytes());
}
}
None => h.update([0]),
}
}
}
h.finalize().into()
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum LawFileError {
NoAuditBlock,
Audit(String),
Declaration {
line: usize,
reason: String,
},
Duplicate {
line: usize,
name: String,
},
InputType {
line: usize,
error: TypeError,
},
}
impl fmt::Display for LawFileError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoAuditBlock => {
write!(f, "law file: no closed `begin audit` ... `end audit` block")
}
Self::Audit(e) => write!(f, "law file: audit block: {e}"),
Self::InputType { line, error } => write!(f, "law file: line {line}: {error}"),
Self::Declaration { line, reason } => write!(f, "law file: line {line}: {reason}"),
Self::Duplicate { line, name } => {
write!(f, "law file: line {line}: `{name}` is declared twice")
}
}
}
}
impl std::error::Error for LawFileError {}
fn check_declaration(line_no: usize, raw: &str, line: &str) -> Result<(), LawFileError> {
let mut head = line.chars();
let reserved = matches!(
(head.next(), head.next()),
(Some(a), Some(b)) if X_LIKE.contains(&a) && DASH_LIKE.contains(&b)
);
if !reserved {
return Ok(());
}
if !raw.starts_with("x-") {
return Err(LawFileError::Declaration {
line: line_no,
reason: "a line starting with `x-` is a declaration: write it in lowercase from the first column".to_owned(),
});
}
let kw_end = line.find(char::is_whitespace).unwrap_or(line.len());
let kw = &line[..kw_end];
if !matches!(kw, "x-input" | "x-metric" | "x-at-least") {
return Err(LawFileError::Declaration {
line: line_no,
reason: format!("unknown declaration `{kw}`"),
});
}
let after = &line[kw_end..];
if !after.starts_with(' ')
|| after[1..].starts_with(char::is_whitespace)
|| after.trim().is_empty()
{
return Err(LawFileError::Declaration {
line: line_no,
reason: format!("`{kw}` must be followed by exactly one space"),
});
}
Ok(())
}
const FORMAT_CHARS: &[(u32, u32)] = &[
(0x00AD, 0x00AD),
(0x0600, 0x0605),
(0x061C, 0x061C),
(0x06DD, 0x06DD),
(0x070F, 0x070F),
(0x0890, 0x0891),
(0x08E2, 0x08E2),
(0x180E, 0x180E),
(0x200B, 0x200F),
(0x202A, 0x202E),
(0x2060, 0x2064),
(0x2066, 0x206F),
(0xFEFF, 0xFEFF),
(0xFFF9, 0xFFFB),
(0x110BD, 0x110BD),
(0x110CD, 0x110CD),
(0x13430, 0x1343F),
(0x1BCA0, 0x1BCA3),
(0x1D173, 0x1D17A),
(0xE0001, 0xE0001),
(0xE0020, 0xE007F),
];
const X_LIKE: &[char] = &[
'\u{58}',
'\u{78}',
'\u{2E3}',
'\u{2093}',
'\u{2169}',
'\u{2179}',
'\u{24CD}',
'\u{24E7}',
'\u{FF38}',
'\u{FF58}',
'\u{1CCED}',
'\u{1D417}',
'\u{1D431}',
'\u{1D44B}',
'\u{1D465}',
'\u{1D47F}',
'\u{1D499}',
'\u{1D4B3}',
'\u{1D4CD}',
'\u{1D4E7}',
'\u{1D501}',
'\u{1D51B}',
'\u{1D535}',
'\u{1D54F}',
'\u{1D569}',
'\u{1D583}',
'\u{1D59D}',
'\u{1D5B7}',
'\u{1D5D1}',
'\u{1D5EB}',
'\u{1D605}',
'\u{1D61F}',
'\u{1D639}',
'\u{1D653}',
'\u{1D66D}',
'\u{1D687}',
'\u{1D6A1}',
'\u{1F147}',
];
const DASH_LIKE: &[char] = &['-', '\u{FE63}', '\u{FF0D}'];
fn allowed_char(c: char) -> bool {
let cp = u32::from(c);
c == '\t'
|| !(c.is_control()
|| c == '\u{2028}'
|| c == '\u{2029}'
|| FORMAT_CHARS.iter().any(|&(lo, hi)| (lo..=hi).contains(&cp)))
}
fn check_text(text: &str) -> Result<(), LawFileError> {
let segments: Vec<&str> = text.split('\n').collect();
let last = segments.len() - 1;
for (n, seg) in segments.iter().enumerate() {
let body = if n < last {
seg.strip_suffix('\r').unwrap_or(seg)
} else {
seg
};
if let Some(c) = body.chars().find(|&c| !allowed_char(c)) {
return Err(LawFileError::Declaration {
line: n + 1,
reason: format!(
"U+{:04X} is not allowed in a law file (only TAB, LF and the CR of a CR LF among control characters, no format or separator character)",
u32::from(c)
),
});
}
}
Ok(())
}
fn metric_line(line: usize, rest: &str) -> Result<(String, MetricExpr), LawFileError> {
let (name, expr) = rest
.split_once('=')
.ok_or_else(|| LawFileError::InputType {
line,
error: TypeError("`x-metric <name> = <expression>` expected".to_owned()),
})?;
let expr = MetricExpr::parse(expr).map_err(|error| LawFileError::InputType { line, error })?;
Ok((name.trim().to_owned(), expr))
}
fn floor_line(line: usize, rest: &str) -> Result<(String, f64), LawFileError> {
let w: Vec<&str> = rest.split_whitespace().collect();
match w.as_slice() {
[name, v] => crate::runtime_parser::law_number(v)
.filter(|v| *v >= 0.0)
.map(|v| ((*name).to_owned(), v)),
_ => None,
}
.ok_or_else(|| LawFileError::InputType {
line,
error: TypeError("`x-at-least <metric> <number>` expected".to_owned()),
})
}
pub fn audit_law_from_file(text: &str) -> Result<crate::audit_law::AuditLaw, LawFileError> {
let mut block: Option<String> = None;
let mut done: Option<String> = None;
let mut inputs = Vec::new();
let mut metrics = Vec::new();
let mut at_least = Vec::new();
let mut at_least_lines: Vec<(usize, String)> = Vec::new();
check_text(text)?;
for (n, raw) in text
.split('\n')
.map(|l| l.strip_suffix('\r').unwrap_or(l))
.enumerate()
{
let line = raw.split('#').next().unwrap_or("").trim();
if let Some(b) = block.as_mut() {
if line == "end audit" {
done = block.take();
} else if !line.is_empty() {
b.push_str(line);
b.push('\n');
}
continue;
}
check_declaration(n + 1, raw.split('#').next().unwrap_or(""), line)?;
if line == "begin audit" {
if done.is_some() {
return Err(LawFileError::Declaration {
line: n + 1,
reason: "a second `begin audit` block".to_owned(),
});
}
block = Some(String::new());
} else if let Some(rest) = line.strip_prefix("x-metric ") {
let (name, expr) = metric_line(n + 1, rest)?;
if metrics
.iter()
.any(|(m, _): &(String, MetricExpr)| *m == name)
{
return Err(LawFileError::Duplicate { line: n + 1, name });
}
metrics.push((name, expr));
} else if let Some(rest) = line.strip_prefix("x-at-least ") {
let (name, v) = floor_line(n + 1, rest)?;
if at_least.iter().any(|(m, _): &(String, f64)| *m == name) {
return Err(LawFileError::Duplicate { line: n + 1, name });
}
at_least_lines.push((n + 1, name.clone()));
at_least.push((name, v));
} else if let Some(rest) = line.strip_prefix("x-input ") {
let rest = rest.trim();
let (name, ty) = rest.split_once(' ').unwrap_or((rest, ""));
let ty = InputType::parse(ty)
.map_err(|error| LawFileError::InputType { line: n + 1, error })?;
if inputs.iter().any(|(m, _): &(String, InputType)| m == name) {
return Err(LawFileError::Duplicate {
line: n + 1,
name: name.to_owned(),
});
}
inputs.push((name.to_owned(), ty));
}
}
if let Some((line, name)) = at_least_lines.iter().find(|(_, name)| {
!metrics
.iter()
.any(|(m, _): &(String, MetricExpr)| m == name)
}) {
return Err(LawFileError::Declaration {
line: *line,
reason: format!("`x-at-least {name}` names no x-metric"),
});
}
let body = done.ok_or(LawFileError::NoAuditBlock)?;
let mut law = crate::runtime_parser::parse_law(&body)
.map_err(|e| LawFileError::Audit(format!("{e:?}")))?;
for (name, ty) in inputs {
law = law.with_input(&name, ty);
}
for (name, expr) in metrics {
law = law.with_metric(&name, expr);
}
for (name, v) in at_least {
law = law.with_at_least(&name, v);
}
Ok(law)
}
fn encode_json(h: &mut Sha256, v: &Json) {
match v {
Json::Null => h.update([0]),
Json::Bool(b) => h.update([1, u8::from(*b)]),
Json::Number(x) => {
h.update([2]);
h.update(x.to_bits().to_le_bytes());
}
Json::Text(t) => {
h.update([3]);
h.update((t.len() as u64).to_le_bytes());
h.update(t.as_bytes());
}
Json::Array(xs) => {
h.update([4]);
h.update((xs.len() as u64).to_le_bytes());
for x in xs {
encode_json(h, x);
}
}
Json::Object(kv) => {
h.update([5]);
h.update((kv.len() as u64).to_le_bytes());
for (k, x) in kv {
h.update((k.len() as u64).to_le_bytes());
h.update(k.as_bytes());
encode_json(h, x);
}
}
}
}
fn input_reading_cases() -> Vec<(&'static str, String, LawKind)> {
use LawKind::{Audit, Quantitative};
const BUILDS: &str = "list of record(features: list of text, id: optional text)";
let v = |value: &str| format!("{{\"v\":{value}}}");
let mut cases: Vec<(&'static str, String, LawKind)> = [
(
BUILDS,
r#"[{"features":["a"],"id":"x"},{"features":["b"]}]"#,
),
(BUILDS, r#"[{"id":"x"},{"features":["b"],"id":"x"}]"#),
(
BUILDS,
r#"[{"features":"a","id":"x"},{"features":["b"],"id":"x"}]"#,
),
(BUILDS, r#"[7,{"features":["b"],"id":"x"}]"#),
(
BUILDS,
r#"[{"features":["a",3],"id":"x"},{"features":["b"],"id":"x"}]"#,
),
(
BUILDS,
r#"[{"features":["a"],"id":5},{"features":["b"],"id":5}]"#,
),
(
BUILDS,
r#"[{"features":["a"],"id":null},{"features":["b"],"id":"x"}]"#,
),
(BUILDS, "[]"),
(BUILDS, r#""std""#),
("list of text", r#"["case-17","case-42"]"#),
("list of text", r#"["case-17",42]"#),
("list of text", "null"),
("number", "1e400"),
("number", "-1e400"),
("number", "true"),
("integer", "3.0"),
("integer", "2.5"),
("text", r#""\u0063ase-17\/\n""#),
("text", r#""\ud83d\ude00""#),
("text", r#""\ud800""#),
("number", "Infinity"),
("number", "NaN"),
]
.iter()
.map(|(t, value)| (*t, v(value), Audit))
.collect();
cases.push(("number", v(&format!("1{}", "0".repeat(399))), Audit));
cases.push(("text", r#"{"v":1,"v":"x"}"#.to_owned(), Audit));
cases.push(("text", r#"{"v":"x","v":1}"#.to_owned(), Audit));
for kind in [Audit, Quantitative] {
cases.push(("number", v(r#""2.5""#), kind));
cases.push(("number", v("2.5"), kind));
}
for depth in [MAX_DEPTH, MAX_DEPTH + 1] {
let inner = depth - 1;
cases.push(("text", v(&("[".repeat(inner) + &"]".repeat(inner))), Audit));
}
cases
}
#[must_use]
pub fn input_reading_fingerprint() -> [u8; 32] {
let mut h = Sha256::new();
h.update(b"lol.input-reading.2");
for (ty, text, kind) in input_reading_cases() {
let t = InputType::parse(ty).expect("fingerprint type parses");
let canon = t.canonical();
h.update((canon.len() as u64).to_le_bytes());
h.update(canon.as_bytes());
h.update((text.len() as u64).to_le_bytes());
h.update(text.as_bytes());
h.update([u8::from(kind == LawKind::Quantitative)]);
match parse_json(&text) {
Err(_) => h.update([0]),
Ok(req) => match read("v", &t, req.get("v"), kind) {
Read::Value(x) => {
h.update([1]);
encode_json(&mut h, x);
}
Read::NotMeasured => h.update([2]),
Read::Rejected(_) => h.update([3]),
},
}
}
h.finalize().into()
}
#[cfg(test)]
mod tests {
use super::*;
const BUILDS: &str = "list of record(features: list of text, id: optional text)";
fn ty(s: &str) -> InputType {
InputType::parse(s).expect("type")
}
fn js(s: &str) -> Json {
parse_json(s).expect("json")
}
#[test]
fn one_bad_part_makes_the_whole_input_not_match() {
let b = ty(BUILDS);
assert!(b.matches(&js(r#"[{"features":["a"],"id":"x"},{"features":[]}]"#)));
for bad in [
r#"[{"features":["a"],"id":"x"},7]"#,
r#"[{"features":["a"],"id":"x"},{"id":"x"}]"#,
r#"[{"features":["a"],"id":"x"},{"features":"b"}]"#,
r#"[{"features":["a"],"id":"x"},{"features":["b",3]}]"#,
r#"[{"features":["a"],"id":"x"},{"features":["b"],"id":5}]"#,
r#""std""#,
] {
assert!(!b.matches(&js(bad)), "{bad}");
}
}
#[test]
fn null_is_not_absent() {
let b = ty(BUILDS);
assert!(b.matches(&js(r#"[{"features":["a"]}]"#)));
assert!(!b.matches(&js(r#"[{"features":["a"],"id":null}]"#)));
}
#[test]
fn audit_and_quantitative_reads_differ_only_for_a_bad_value() {
let t = ty("number");
let good = js("2.5");
let bad = js("\"2.5\"");
assert_eq!(
read("x", &t, Some(&good), LawKind::Audit),
Read::Value(&good)
);
assert_eq!(
read("x", &t, Some(&good), LawKind::Quantitative),
Read::Value(&good)
);
assert_eq!(read("x", &t, Some(&bad), LawKind::Audit), Read::NotMeasured);
assert!(matches!(
read("x", &t, Some(&bad), LawKind::Quantitative),
Read::Rejected(_)
));
assert_eq!(
read("x", &t, None, LawKind::Quantitative),
Read::NotMeasured
);
}
#[test]
fn the_canonical_text_reads_back_to_the_same_type() {
for s in [
BUILDS,
"list of list of integer",
"record( a:text ,b: optional number )",
] {
let t = ty(s);
assert_eq!(ty(&t.canonical()), t, "{s}");
}
assert_eq!(
ty("record( a:text ,b: optional number )").canonical(),
"record(a: text, b: optional number)"
);
}
#[test]
fn malformed_types_are_rejected() {
for bad in [
"",
"texts",
"list text",
"record(a text)",
"record(a: text",
"record(a: text, a: text)",
"record()",
"text text",
"list of",
"record(: text)",
] {
assert!(InputType::parse(bad).is_err(), "{bad:?}");
}
}
#[test]
fn json_edge_cases() {
assert_eq!(js(r#"{"a":1,"a":2}"#).get("a"), Some(&Json::Number(2.0)));
assert_eq!(js(r#""\ud83d\ude00""#), Json::Text("\u{1f600}".to_owned()));
assert_eq!(js(" [ ] "), Json::Array(Vec::new()));
for bad in [
"",
"01",
"1.",
"-",
"[1,]",
"{\"a\"}",
"\"\\ud800\"",
"nul",
"[1] 2",
"\"a\u{1}\"",
] {
assert!(parse_json(bad).is_err(), "{bad:?}");
}
let deep = "[".repeat(MAX_DEPTH + 1) + &"]".repeat(MAX_DEPTH + 1);
assert!(parse_json(&deep).is_err());
}
#[test]
fn each_fingerprint_case_reads_as_decided() {
let expected = [
"value", "not", "not", "not", "not", "not", "not", "value", "not", "value", "not", "not", "not", "not", "not", "value", "not", "value", "value", "error", "error", "error", "not", "value", "not", "not", "value", "rejected",
"value", "not", "error", ];
let got: Vec<&str> = input_reading_cases()
.into_iter()
.map(|(ty, text, kind)| {
let t = InputType::parse(ty).expect("type");
parse_json(&text).map_or("error", |req| match read("v", &t, req.get("v"), kind) {
Read::Value(_) => "value",
Read::NotMeasured => "not",
Read::Rejected(_) => "rejected",
})
})
.collect();
assert_eq!(got, expected);
}
type Row = (
Option<f64>,
Option<f64>,
Option<f64>,
Option<f64>,
Option<Vec<&'static str>>,
);
#[test]
fn each_derivation_case_measures_as_decided() {
let none = None;
let expected: [Row; 13] = [
(Some(2.0), Some(2.0), Some(1.0), none, None),
(Some(2.0), Some(0.0), Some(2.0), none, None), (Some(2.0), Some(0.0), Some(1.0), none, None), (Some(2.0), Some(2.0), Some(2.0), none, None), (Some(2.0), Some(2.0), none, none, None), (Some(2.0), Some(0.0), Some(1.0), none, None), (Some(0.0), Some(0.0), Some(0.0), none, None), (none, Some(0.0), none, none, None), (none, Some(0.0), none, none, None), (none, Some(0.0), none, none, None), (Some(1.0), Some(0.0), Some(1.0), none, None), (none, Some(0.0), none, none, None), (
Some(0.0),
Some(0.0),
Some(0.0),
Some(4.0),
Some(vec!["p", "q"]),
),
];
for ((law_text, inputs), want) in DERIVATION_CASES.iter().zip(expected) {
let law = audit_law_from_file(law_text).expect("law");
let m = measurements(&law, &parse_json(inputs).expect("inputs"));
let k = m
.range("k")
.map(|r| r.iter().map(String::as_str).collect::<Vec<_>>());
let got = (
m.number("count"),
m.number("sets"),
m.number("ids"),
m.number("n"),
k,
);
assert_eq!(got, want, "{inputs}");
}
}
#[test]
fn a_declaration_made_twice_does_not_read() {
let law = |extra: &str| {
audit_law_from_file(&format!(
"x-input b list of record(f: list of text)\nx-metric n = count(b)\nx-at-least n 1\n{extra}begin audit\naudit a\nevidence n\nend audit\n"
))
};
assert!(law("").is_ok());
for extra in [
"x-metric n = count(b)\n",
"x-metric n = distinct(set(b[].f))\n",
"x-at-least n 1\n",
"x-at-least n 2\n",
"x-input b list of text\n",
] {
assert!(
matches!(law(extra), Err(LawFileError::Duplicate { .. })),
"{extra}"
);
}
}
#[test]
fn a_declaration_line_reads_only_in_its_one_form() {
let ok =
"x-input b list of record(f: list of text)\nx-metric n = count(b)\nx-at-least n 1\n";
let law = |decls: &str| {
audit_law_from_file(&format!(
"{decls}begin audit\naudit a\nevidence n\nend audit\n"
))
};
assert!(law(ok).is_ok());
assert!(law(&format!("{ok}# x-foo in a comment\n # X-Metric too\n")).is_ok());
assert!(matches!(
law(&ok.replace("x-at-least n 1", "x-at-least n -1")),
Err(LawFileError::InputType { .. })
));
assert!(law(&ok.replace("x-at-least n 1", "x-at-least n -0")).is_ok());
for bad in [
ok.replace("x-metric n", "x-metric\tn"),
ok.replace("x-input b", "x-input\tb"),
ok.replace("x-at-least n", "x-at-least\tn"),
ok.replace("x-metric n", "x-metric n"),
ok.replace("x-metric n", "x-metric\u{a0}n"),
format!("{ok}x-foo n 1\n"),
format!("{ok}x-metrics t = count(b)\n"),
format!("{ok}x-at-least t 2\n"),
format!("{ok}x-metric\n"),
format!(" {ok}"),
format!("\t{ok}"),
ok.replace("x-metric n", "X-metric n"),
ok.replace("x-at-least", "X-AT-LEAST"),
format!("{ok}X-foo 1\n"),
] {
assert!(
matches!(law(&bad), Err(LawFileError::Declaration { .. })),
"{bad:?}: {:?}",
law(&bad)
);
}
}
#[test]
fn only_tab_lf_and_cr_lf_among_controls_and_no_format_characters() {
let ok = "x-input b list of record(f: list of text)\nx-metric n = count(b)\nbegin audit\naudit a\nevidence n\nend audit\n";
assert!(audit_law_from_file(ok).is_ok());
assert!(audit_law_from_file(&ok.replace('\n', "\r\n")).is_ok());
assert!(audit_law_from_file(&format!("{ok}#\tnote\n")).is_ok());
for bad in [
format!("\u{feff}{ok}"),
ok.replace("x-metric", "\u{feff}x-metric"),
ok.replace('\n', "\r"),
ok.replacen('\n', "\r", 1),
format!("{ok}\r"),
ok.replacen('\n', "\u{b}", 1),
ok.replacen('\n', "\u{c}", 1),
ok.replacen('\n', "\u{1c}", 1),
ok.replacen('\n', "\u{85}", 1),
ok.replacen('\n', "\u{2028}", 1),
ok.replacen('\n', "\u{2029}", 1),
format!("{ok}# a\u{200b}b\n"),
format!("{ok}# nul \u{0}\n"),
ok.replace("x-metric", "\u{ff58}-metric"),
ok.replace("x-metric", "x\u{ff0d}metric"),
ok.replace("x-metric", "\u{2179}-metric"),
] {
assert!(
matches!(
audit_law_from_file(&bad),
Err(LawFileError::Declaration { .. })
),
"{bad:?}"
);
}
}
#[test]
fn metric_expressions_parse_and_read_back() {
for (text, canon) in [
("count(b)", "count(b)"),
(" distinct( b[].id ) ", "distinct(b[].id)"),
("distinct(set(b[].f))", "distinct(set(b[].f))"),
] {
let e = MetricExpr::parse(text).expect(text);
assert_eq!(e.canonical(), canon);
assert_eq!(MetricExpr::parse(&e.canonical()), Ok(e));
}
for bad in [
"count()",
"count(b[].f)",
"distinct(b)",
"distinct(set(b))",
"sum(b)",
"distinct(b[].)",
"distinct(set(b[].f)",
] {
assert!(MetricExpr::parse(bad).is_err(), "{bad}");
}
}
#[test]
fn the_fingerprint_cases_all_parse() {
let _ = input_reading_fingerprint();
}
#[test]
fn overflowing_literals_are_infinite_and_not_numbers() {
assert_eq!(parse_json("1e400"), Ok(Json::Number(f64::INFINITY)));
assert_eq!(parse_json("-1e400"), Ok(Json::Number(f64::NEG_INFINITY)));
assert!(!InputType::Number.matches(&Json::Number(f64::INFINITY)));
}
}