mod fold;
mod limits;
mod place;
use std::borrow::Cow;
use std::collections::BTreeSet;
use std::path::Path;
use jsonschema::error::ValidationErrorKind;
use jsonschema::{PatternOptions, ReferencingError, ValidationError, ValidationOptions, Validator};
use jsonvet::{Dialect, Event, Severity};
use serde_json::{Map, Value};
use crate::settings::Settings;
use fold::documents;
use limits::{MAX_SCHEMA_DEPTH, Reach, registry};
use place::{place, place_found, pointer_path, segments};
const MAX_ERRORS: usize = 100;
const MAX_MESSAGE: usize = 200;
const BACKTRACK_LIMIT: usize = 5_000;
const NOT_CHECKS: [&str; 9] = [
"$schema",
"$id",
"id",
"$defs",
"definitions",
"$anchor",
"$dynamicAnchor",
"$vocabulary",
"$comment",
];
const ON_CHILDREN: [&str; 13] = [
"allOf",
"anyOf",
"oneOf",
"not",
"if",
"then",
"else",
"$ref",
"$dynamicRef",
"$recursiveRef",
"dependentSchemas",
"dependencies",
"extends",
];
const ON_MEMBERS: [&str; 3] = ["properties", "patternProperties", "additionalProperties"];
const EACH: &str = "jsonvet-each";
pub(crate) struct Schema {
validator: Validator,
items: Option<Split>,
members: Option<Split>,
key_patterns: Vec<(String, Validator)>,
string_patterns: Vec<(String, Validator)>,
}
struct Split {
whole: Validator,
each: Validator,
}
fn children(value: &Value) -> impl Iterator<Item = (Cow<'_, str>, &Value)> {
let members = value.as_object().into_iter().flatten();
let items = value.as_array().into_iter().flatten().enumerate();
members
.map(|(key, member)| (Cow::Borrowed(key.as_str()), member))
.chain(items.map(|(i, item)| (Cow::Owned(i.to_string()), item)))
}
fn backtracked(error: &ValidationError<'_>) -> bool {
match error.kind() {
ValidationErrorKind::BacktrackLimitExceeded { .. } => true,
ValidationErrorKind::PropertyNames { error } => backtracked(error),
_ => false,
}
}
fn patterns<'v>(
schema: &'v Value,
names: bool,
keys: &mut BTreeSet<&'v str>,
strings: &mut BTreeSet<&'v str>,
) {
for (key, member) in children(schema) {
match (key.as_ref(), member) {
("patternProperties", Value::Object(regexes)) => {
keys.extend(regexes.keys().map(String::as_str));
}
("pattern", Value::String(regex)) => {
strings.insert(regex);
if names {
keys.insert(regex);
}
}
_ => {}
}
patterns(member, names || key == "propertyNames", keys, strings);
}
}
struct Found {
path: Vec<String>,
keys: Vec<String>,
key: bool,
message: String,
}
#[derive(Default)]
struct Errors {
kept: Vec<Found>,
more: bool,
}
impl Errors {
fn add(&mut self, found: impl IntoIterator<Item = Found>) -> bool {
for found in found {
if self.kept.len() == MAX_ERRORS {
self.more = true;
break;
}
self.kept.push(found);
}
!self.more
}
}
fn found(index: Option<usize>, value: &Value, error: &ValidationError<'_>) -> Vec<Found> {
if backtracked(error) {
return Vec::new();
}
let path = pointer_path(index, error);
if let Some(keys) = unexpected(value, error) {
return keys
.into_iter()
.take(MAX_ERRORS + 1)
.map(|key| Found {
path: path.clone(),
message: format!("Additional properties are not allowed ('{key}' was unexpected)"),
keys: vec![key],
key: true,
})
.collect();
}
let keys = match error.kind() {
ValidationErrorKind::UnevaluatedProperties { unexpected } => unexpected.clone(),
ValidationErrorKind::PropertyNames { error } => error
.instance()
.as_str()
.map(|key| vec![key.to_owned()])
.unwrap_or_default(),
_ => Vec::new(),
};
vec![Found {
path,
keys,
key: false,
message: message(error),
}]
}
fn unexpected(value: &Value, error: &ValidationError<'_>) -> Option<Vec<String>> {
match error.kind() {
ValidationErrorKind::AdditionalProperties { unexpected } => Some(unexpected.clone()),
ValidationErrorKind::FalseSchema
if error
.schema_path()
.as_str()
.ends_with("/additionalProperties") =>
{
let object = value.pointer(error.instance_path().as_str())?;
let members = object.as_object()?;
(object != error.instance().as_ref()).then(|| members.keys().cloned().collect())
}
_ => None,
}
}
impl Schema {
pub(crate) fn load(path: &Path, settings: &Settings) -> Result<Schema, String> {
let name = path.to_string_lossy();
let text = crate::read_small(path, crate::MAX_SCHEMA_BYTES)
.map_err(|e| format!("cannot read the schema {name}: {e}"))?;
let mut config = settings.config_for(&name);
if config.dialect() == Dialect::JsonLines
&& Dialect::for_path(path) != Some(Dialect::JsonLines)
{
config = settings.config(Dialect::Json5);
}
let config = config.max_depth(MAX_SCHEMA_DEPTH);
let (report, events) = jsonvet::parse(&text, &config);
let at = |offset, problem: &dyn std::fmt::Display| {
let at = report.location(offset);
format!("the schema {name}:{}:{}: {problem}", at.line, at.column)
};
if let Some(problem) = report
.problems()
.iter()
.find(|p| p.severity() == Severity::Error)
{
return Err(at(problem.offset(), problem));
}
let events = events.unwrap_or_default();
let mut documents = documents(&events);
if documents.len() != 1 {
return Err(format!("the schema {name} must hold one value"));
}
let document = documents.swap_remove(0);
let events = &events[document.events.clone()];
if let Some((offset, reason)) = document.unchecked {
return Err(at(offset, &reason));
}
let schema = &document.value;
let outside = |error: &ReferencingError| match error {
ReferencingError::Unretrievable { uri, .. } => Some(format!(
"the schema {name} has a `$ref` to {uri}, outside the file; jsonvet fetches no \
schemas"
)),
_ => None,
};
let registry = registry(schema)
.map_err(|error| outside(&error).unwrap_or_else(|| at(events[0].offset, &error)))?;
let kept = Reach::new(schema).measure(®istry).map_err(|over| {
let (offset, _) = place(events, &[(over.path.clone(), vec![over.key.clone()])])[0];
at(offset, &over.message())
})?;
let options = jsonschema::options()
.offline()
.with_pattern_options(PatternOptions::fancy_regex().backtrack_limit(BACKTRACK_LIMIT));
let error = match options.build(schema) {
Ok(validator) => {
let (mut keys, mut strings) = (BTreeSet::new(), BTreeSet::new());
patterns(schema, false, &mut keys, &mut strings);
let compile = |regexes: BTreeSet<&str>| {
regexes
.into_iter()
.filter_map(|regex| {
let schema = serde_json::json!({ "pattern": regex });
Some((regex.to_owned(), options.build(&schema).ok()?))
})
.collect()
};
return Ok(Schema {
validator,
items: split(schema, kept, false, &options),
members: split(schema, kept, true, &options),
key_patterns: compile(keys),
string_patterns: compile(strings),
});
}
Err(error) => error,
};
if let ValidationErrorKind::Referencing(error) = error.kind()
&& let Some(message) = outside(error)
{
return Err(message);
}
let (offset, _) = place_found(events, &found(None, schema, &error))[0];
Err(at(offset, &error))
}
pub(crate) fn errors(&self, events: &[Event<'_>]) -> Vec<(usize, String, bool)> {
let mut errors = Vec::new();
for document in documents(events) {
if let Some((offset, reason)) = document.unchecked {
errors.push((
offset,
format!("{reason}, so this document is not checked against the schema"),
false,
));
continue;
}
let found = self.collect(&document.value);
if found.kept.is_empty() {
continue;
}
let events = &events[document.events];
let spots = place_found(events, &found.kept);
let mut placed: Vec<(usize, String, bool)> = found
.kept
.into_iter()
.zip(spots)
.map(|(found, (offset, at))| {
(offset, cut(format!("{} (at {at})", found.message)), false)
})
.collect();
placed.sort_by_key(|(offset, ..)| *offset);
errors.append(&mut placed);
if found.more {
errors.push((
events[0].offset,
"... and more schema errors in this document".to_owned(),
true,
));
}
}
errors
}
fn collect(&self, value: &Value) -> Errors {
let mut errors = Errors::default();
if !(self.key_patterns.is_empty() && self.string_patterns.is_empty()) {
self.unmatched(value, &mut Vec::new(), &mut errors);
}
if errors.more || self.validator.is_valid(value) {
return errors;
}
match (value, &self.items, &self.members) {
(Value::Array(items), Some(Split { whole, each }), _) => {
let mut after = Vec::new();
for error in whole.iter_errors(value) {
let found = found(None, value, &error);
let keyword = segments(error.schema_path().as_str()).next();
if keyword.is_some_and(|k| {
["contains", "minContains", "maxContains"].contains(&k.as_str())
}) {
after.extend(found);
} else if !errors.add(found) {
return errors;
}
}
for (i, item) in items.iter().enumerate() {
for error in each.iter_errors(item) {
if !errors.add(found(Some(i), item, &error)) {
return errors;
}
}
}
errors.add(after);
}
(Value::Object(members), _, Some(Split { whole, each })) => {
for error in whole.iter_errors(value) {
if !errors.add(found(None, value, &error)) {
return errors;
}
}
for (key, member) in members {
let one = Value::Object(Map::from_iter([(key.clone(), member.clone())]));
for error in each.iter_errors(&one) {
if !errors.add(found(None, &one, &error)) {
return errors;
}
}
}
}
_ => {
for error in self.validator.iter_errors(value) {
if !errors.add(found(None, value, &error)) {
break;
}
}
}
}
errors
}
fn unmatched(&self, value: &Value, path: &mut Vec<String>, errors: &mut Errors) {
let fails = |regex: &Validator, value: &Value| {
regex
.validate(value)
.is_err_and(|error| backtracked(&error))
};
let message = |what: &str, regex: &str| {
format!(
"this {what} could not be checked against the pattern \"{regex}\": matching it \
takes more than {BACKTRACK_LIMIT} backtracking steps"
)
};
if value.is_string() {
for (regex, validator) in &self.string_patterns {
if fails(validator, value)
&& !errors.add([Found {
path: path.clone(),
keys: Vec::new(),
key: false,
message: message("string", regex),
}])
{
return;
}
}
}
for (key, member) in children(value) {
if value.is_object() {
let name = Value::String(key.clone().into_owned());
for (regex, validator) in &self.key_patterns {
if fails(validator, &name)
&& !errors.add([Found {
path: path.clone(),
keys: vec![key.clone().into_owned()],
key: true,
message: message("key", regex),
}])
{
return;
}
}
}
path.push(key.into_owned());
self.unmatched(member, path, errors);
path.pop();
if errors.more {
return;
}
}
}
}
fn split(schema: &Value, kept: bool, members: bool, options: &ValidationOptions) -> Option<Split> {
let all = schema.as_object()?;
let (moved, also): (&[&str], &[&str]) = if members {
(&ON_MEMBERS, &["unevaluatedProperties"])
} else {
(&["items"], &["prefixItems", "unevaluatedItems"])
};
let checks = |key: &&str| all.contains_key(*key);
if !kept || ON_CHILDREN.iter().chain(also).any(checks) || !moved.iter().any(checks) {
return None;
}
let mut whole = all.clone();
let mut part = Map::new();
for key in moved {
if let Some(value) = whole.remove(*key) {
part.insert((*key).to_owned(), value);
}
}
let part = match (members, part.remove("items")) {
(false, Some(items)) if items.is_object() || items.is_boolean() => items,
(false, _) => return None,
(true, _) => Value::Object(part),
};
let mut each: Map<String, Value> = all
.iter()
.filter(|(key, _)| NOT_CHECKS.contains(&key.as_str()))
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
each.insert(EACH.to_owned(), part);
each.insert(
"allOf".to_owned(),
serde_json::json!([{ "$ref": format!("#/{EACH}") }]),
);
Some(Split {
whole: options.build(&Value::Object(whole)).ok()?,
each: options.build(&Value::Object(each)).ok()?,
})
}
fn message(error: &ValidationError<'_>) -> String {
if fits(error.instance()) {
let message = error.to_string();
if message.len() <= MAX_MESSAGE {
return message;
}
}
error.masked().to_string()
}
fn fits(value: &Value) -> bool {
struct Budget(usize);
impl std::io::Write for Budget {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0 = self
.0
.checked_sub(bytes.len())
.ok_or(std::io::ErrorKind::WriteZero)?;
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
serde_json::to_writer(Budget(MAX_MESSAGE), value).is_ok()
}
fn cut(mut message: String) -> String {
if message.len() > MAX_MESSAGE {
let mut end = MAX_MESSAGE;
while !message.is_char_boundary(end) {
end -= 1;
}
message.truncate(end);
message.push_str("...");
}
message
}
#[cfg(test)]
mod tests {
use super::limits::{Reach, registry};
use super::{MAX_MESSAGE, Schema, cut, fits, message, split};
use jsonvet::{Config, Dialect};
use proptest::prelude::*;
use serde_json::{Value, json};
use super::fold::tests::any_json;
#[test]
fn a_large_nested_value_is_named_not_quoted() {
let validator = jsonschema::validator_for(&json!({"type": "string"})).expect("a schema");
let long = "x".repeat(300);
for instance in [
json!([{"a": [long.clone()]}]),
json!({"a": {"b": long.clone()}}),
json!({ long.clone(): 1 }),
] {
let error = validator.iter_errors(&instance).next().expect("an error");
assert_eq!(message(&error), "value is not of type \"string\"");
}
let small = json!([{"a": [1]}]);
let error = validator.iter_errors(&small).next().expect("an error");
assert_eq!(message(&error), "[{\"a\":[1]}] is not of type \"string\"");
}
#[test]
fn a_value_fits_while_its_json_takes_at_most_the_budget() {
let x = |n| json!("x".repeat(n));
assert!(fits(&x(MAX_MESSAGE - 2)));
assert!(!fits(&x(MAX_MESSAGE - 1)));
}
#[test]
fn a_long_message_is_cut_at_a_character_boundary() {
let long = format!("x{}", "\u{e9}".repeat(200));
let message = cut(long.clone());
assert_eq!(message.len(), MAX_MESSAGE - 1 + "...".len());
assert!(message.ends_with("\u{e9}..."), "{message}");
assert_eq!(
cut(long[..MAX_MESSAGE - 1].to_owned()),
long[..MAX_MESSAGE - 1]
);
let full = "x".repeat(MAX_MESSAGE);
assert_eq!(cut(full.clone()), full);
}
fn split_schemas() -> [(Value, bool); 10] {
[
(
json!({"items": {"type": "string"}, "minItems": 2, "uniqueItems": true}),
true,
),
(
json!({
"items": {"$ref": "#/$defs/p"},
"contains": {"type": "null"},
"minContains": 2,
"$defs": {"p": {"properties": {"a": {"type": "integer"}}, "additionalProperties": false}},
}),
true,
),
(
json!({
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://example.com/r.json",
"items": {"$ref": "#/definitions/n"},
"definitions": {"n": {"type": ["array", "integer"], "items": {"$ref": "#/definitions/n"}}},
}),
true,
),
(
json!({"items": {"additionalProperties": false}, "maxItems": 3}),
true,
),
(json!({"items": false}), true),
(json!({"items": {"$ref": "#"}, "type": "array"}), false),
(
json!({"items": {"anyOf": [{"$ref": "#"}]}, "type": "array"}),
false,
),
(
json!({"items": {"type": "string"}, "allOf": [{"items": {"minLength": 1}}]}),
false,
),
(
json!({"$schema": "https://json-schema.org/draft/2019-09/schema", "items": [{"type": "string"}]}),
false,
),
(
json!({"items": {"type": "string"}, "$defs": {"a": {"$ref": "#/items"}}, "contains": {"$ref": "#/$defs/a"}}),
false,
),
]
}
type Errors = Vec<(usize, String, bool)>;
fn one_pass_and_split(schema: &Value, text: &str) -> (Errors, Option<Errors>) {
let options = jsonschema::options();
let checked = |items, members| {
let schema = Schema {
validator: options.build(schema).expect("a schema"),
items,
members,
key_patterns: Vec::new(),
string_patterns: Vec::new(),
};
let (_, events) = jsonvet::parse(text, &Config::new(Dialect::Json));
schema.errors(&events.unwrap_or_default())
};
let registry = registry(schema).expect("a registry");
let kept = Reach::new(schema).measure(®istry).is_ok_and(|kept| kept);
let items = split(schema, kept, false, &options);
let members = split(schema, kept, true, &options);
let split = (items.is_some() || members.is_some()).then(|| checked(items, members));
(checked(None, None), split)
}
#[test]
fn a_split_schema_reports_what_one_pass_does_past_the_cap() {
let text = format!("[{}]", vec!["{\"a\": 1, \"b\": 1}"; 150].join(","));
for (schema, splits) in split_schemas() {
let (one_pass, split) = one_pass_and_split(&schema, &text);
assert_eq!(split.is_some(), splits, "{schema}");
if let Some(split) = split {
assert_eq!(split, one_pass, "{schema}");
}
}
}
fn member_schemas() -> [(Value, bool); 7] {
[
(
json!({"properties": {"a": {"type": "string"}, "b": {"const": 1}}, "required": ["c"], "maxProperties": 2}),
true,
),
(
json!({"patternProperties": {"^a": {"type": "integer"}}, "additionalProperties": false, "propertyNames": {"maxLength": 2}}),
true,
),
(
json!({"additionalProperties": false, "minProperties": 1}),
true,
),
(
json!({"additionalProperties": {"$ref": "#/$defs/n"}, "$defs": {"n": {"type": ["object", "null"], "additionalProperties": {"$ref": "#/$defs/n"}}}}),
true,
),
(
json!({"properties": {"a": true}, "unevaluatedProperties": false}),
false,
),
(
json!({"properties": {"a": {"type": "string"}}, "anyOf": [{"required": ["a"]}]}),
false,
),
(
json!({"additionalProperties": {"$ref": "#/properties/a"}, "properties": {"a": {"type": "null"}}}),
false,
),
]
}
#[test]
fn a_split_object_schema_reports_as_many_errors_past_the_cap() {
let members: Vec<String> = (0..150)
.map(|i| format!("\"a{i}\": {{\"b\": 1}}"))
.collect();
let text = format!("{{{}}}", members.join(","));
for (schema, splits) in member_schemas() {
let (one_pass, split) = one_pass_and_split(&schema, &text);
assert_eq!(split.is_some(), splits, "{schema}");
if let Some(split) = split {
assert_eq!(split.len(), one_pass.len(), "{schema}");
assert_eq!(split.last(), one_pass.last(), "{schema}");
}
}
}
proptest! {
#[test]
fn a_split_object_schema_reports_what_one_pass_does(members in prop::collection::vec(("[ab]{1,2}", any_json()), 0..6)) {
let object: serde_json::Map<String, Value> = members.into_iter().collect();
let text = serde_json::to_string(&object).unwrap_or_default();
for (schema, _) in member_schemas() {
let (one_pass, split) = one_pass_and_split(&schema, &text);
if let Some(split) = split {
prop_assert_eq!(split, one_pass, "{}", schema);
}
}
}
#[test]
fn a_split_schema_reports_what_one_pass_does(items in prop::collection::vec(any_json(), 0..6)) {
let text = serde_json::to_string(&items).unwrap_or_default();
for (schema, _) in split_schemas() {
let (one_pass, split) = one_pass_and_split(&schema, &text);
if let Some(split) = split {
prop_assert_eq!(split, one_pass, "{}", schema);
}
}
}
}
}