use std::sync::Arc;
use pictor_runtime::constrained_decoding::TokenConstraint;
use pictor_runtime::grammar::{
compile_json_schema, compile_json_schema_str, EarleyRecognizer, GrammarConstraint,
JsonSchemaCompileError,
};
fn ascii_constraint_from_schema(schema_json: &str) -> GrammarConstraint {
let grammar = compile_json_schema_str(schema_json).expect("schema must compile");
GrammarConstraint::new(
grammar,
|id| if id < 128 { vec![id as u8] } else { vec![] },
128,
)
}
fn feed_str_constraint(c: &mut GrammarConstraint, s: &str) -> bool {
for b in s.bytes() {
if !c.advance(b as u32) {
return false;
}
}
true
}
fn recognizer_from_schema(schema_json: &str) -> EarleyRecognizer {
let mut g = compile_json_schema_str(schema_json).expect("schema must compile");
g.normalise_terminals();
EarleyRecognizer::new(Arc::new(g))
}
fn feed_str_recognizer(rec: &mut EarleyRecognizer, s: &str) -> bool {
for b in s.bytes() {
if !rec.feed_byte(b) {
return false;
}
}
true
}
#[test]
fn compile_empty_object_schema() {
let result = compile_json_schema_str("{}");
assert!(
result.is_ok(),
"empty schema should compile successfully, got: {:?}",
result.err()
);
let g = result.unwrap();
assert!(!g.rules.is_empty(), "compiled grammar must have rules");
}
#[test]
fn compile_string_type() {
let g = compile_json_schema_str(r#"{"type":"string"}"#).expect("should compile");
assert!(!g.rules.is_empty());
let start_rules: Vec<_> = g.rules_for(g.start()).collect();
assert!(
!start_rules.is_empty(),
"start NT must have at least one rule"
);
}
#[test]
fn compile_integer_type() {
let mut rec = recognizer_from_schema(r#"{"type":"integer"}"#);
assert!(
feed_str_recognizer(&mut rec, "42"),
"integer 42 should be accepted"
);
assert!(rec.is_accepting(), "42 is a complete integer");
let mut rec2 = recognizer_from_schema(r#"{"type":"integer"}"#);
assert!(feed_str_recognizer(&mut rec2, "-7"));
assert!(rec2.is_accepting());
}
#[test]
fn compile_number_type() {
let mut rec = recognizer_from_schema(r#"{"type":"number"}"#);
assert!(
feed_str_recognizer(&mut rec, "3"),
"whole number should be accepted"
);
assert!(rec.is_accepting());
let mut rec2 = recognizer_from_schema(r#"{"type":"number"}"#);
assert!(feed_str_recognizer(&mut rec2, "3.14"));
assert!(rec2.is_accepting(), "3.14 must be accepted as a number");
let mut rec3 = recognizer_from_schema(r#"{"type":"number"}"#);
assert!(feed_str_recognizer(&mut rec3, "-0.5"));
assert!(rec3.is_accepting());
}
#[test]
fn compile_boolean_type() {
let mut rec_true = recognizer_from_schema(r#"{"type":"boolean"}"#);
assert!(feed_str_recognizer(&mut rec_true, "true"));
assert!(rec_true.is_accepting());
let mut rec_false = recognizer_from_schema(r#"{"type":"boolean"}"#);
assert!(feed_str_recognizer(&mut rec_false, "false"));
assert!(rec_false.is_accepting());
let mut rec_bad = recognizer_from_schema(r#"{"type":"boolean"}"#);
let ok = feed_str_recognizer(&mut rec_bad, "yes");
assert!(!ok || !rec_bad.is_accepting(), "\"yes\" is not a boolean");
}
#[test]
fn compile_null_type() {
let mut rec = recognizer_from_schema(r#"{"type":"null"}"#);
assert!(feed_str_recognizer(&mut rec, "null"));
assert!(rec.is_accepting());
let mut rec_bad = recognizer_from_schema(r#"{"type":"null"}"#);
let ok = feed_str_recognizer(&mut rec_bad, "undefined");
assert!(!ok || !rec_bad.is_accepting());
}
#[test]
fn compile_object_with_required_props() {
let schema = r#"{
"type": "object",
"properties": {
"x": {"type": "integer"},
"y": {"type": "string"}
},
"required": ["x", "y"]
}"#;
let g = compile_json_schema_str(schema).expect("should compile");
assert!(
g.nt_count >= 3,
"must have multiple NTs for object with properties"
);
assert!(g.rules.len() >= 2);
}
#[test]
fn compile_array_of_integers() {
let schema = r#"{"type":"array","items":{"type":"integer"}}"#;
let mut rec = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec, "[]"));
assert!(rec.is_accepting());
let mut rec2 = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec2, "[42]"));
assert!(rec2.is_accepting());
let mut rec3 = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec3, "[1,2,3]"));
assert!(rec3.is_accepting());
}
#[test]
fn compile_nested_object_array() {
let schema = r#"{
"type": "object",
"properties": {
"items": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {"type": "integer"}
},
"required": ["id"]
}
}
},
"required": ["items"]
}"#;
let g = compile_json_schema_str(schema).expect("nested schema should compile");
assert!(g.nt_count >= 4, "expected multiple NTs for 3-level nesting");
assert!(!g.rules.is_empty());
}
#[test]
fn compile_enum_strings() {
let schema = r#"{"enum":["foo","bar","baz"]}"#;
let mut c = ascii_constraint_from_schema(schema);
let mask = c.allowed_tokens(&[], 128).unwrap();
assert!(
mask[b'"' as usize],
"quote should be allowed at start of enum string"
);
assert!(
!mask[b'x' as usize],
"'x' not a valid start for any enum value"
);
assert!(feed_str_constraint(&mut c, r#""foo""#));
assert!(c.is_complete());
let mut c2 = ascii_constraint_from_schema(schema);
let ok = feed_str_constraint(&mut c2, r#""qux""#);
assert!(!ok || !c2.is_complete());
}
#[test]
fn compile_enum_integers() {
let schema = r#"{"enum":[1,2,3]}"#;
let mut rec = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec, "2"));
assert!(rec.is_accepting());
let mut rec2 = recognizer_from_schema(schema);
let ok = feed_str_recognizer(&mut rec2, "4");
assert!(!ok || !rec2.is_accepting());
}
#[test]
fn compile_any_of() {
let schema = r#"{"anyOf":[{"type":"string"},{"type":"integer"}]}"#;
let mut rec_str = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec_str, r#""hello""#));
assert!(rec_str.is_accepting());
let mut rec_int = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec_int, "99"));
assert!(rec_int.is_accepting());
let mut rec_bad = recognizer_from_schema(schema);
let ok = feed_str_recognizer(&mut rec_bad, "true");
assert!(!ok || !rec_bad.is_accepting());
}
#[test]
fn compile_one_of() {
let schema = r#"{"oneOf":[{"type":"boolean"},{"type":"null"}]}"#;
let mut rec_true = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec_true, "true"));
assert!(rec_true.is_accepting());
let mut rec_null = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec_null, "null"));
assert!(rec_null.is_accepting());
let mut rec_int = recognizer_from_schema(schema);
let ok = feed_str_recognizer(&mut rec_int, "42");
assert!(!ok || !rec_int.is_accepting());
}
#[test]
fn compile_ref_to_defs() {
let schema = r##"{
"$defs": {
"MyString": {"type": "string"}
},
"$ref": "#/$defs/MyString"
}"##;
let g = compile_json_schema_str(schema).expect("$ref to $defs must compile");
assert!(!g.rules.is_empty());
let mut rec = EarleyRecognizer::new({
let mut g2 = g;
g2.normalise_terminals();
Arc::new(g2)
});
assert!(feed_str_recognizer(&mut rec, r#""hello""#));
assert!(rec.is_accepting());
}
#[test]
fn compile_recursive_ref() {
let schema = r##"{
"$defs": {
"Node": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"next": {
"anyOf": [
{"$ref": "#/$defs/Node"},
{"type": "null"}
]
}
},
"required": ["value", "next"]
}
},
"$ref": "#/$defs/Node"
}"##;
let result = compile_json_schema_str(schema);
assert!(
result.is_ok(),
"recursive $ref schema must compile, got: {:?}",
result.err()
);
let g = result.unwrap();
assert!(!g.rules.is_empty());
}
#[test]
fn unsupported_keyword_not() {
let schema = r#"{"not": {"type": "string"}}"#;
let err = compile_json_schema_str(schema).expect_err("'not' should not be supported");
assert!(
matches!(err, JsonSchemaCompileError::UnsupportedKeyword(ref kw) if kw == "not"),
"expected UnsupportedKeyword(\"not\"), got: {err:?}"
);
}
#[test]
fn unsupported_keyword_if_then() {
let schema = r#"{"if": {"type": "string"}, "then": {"type": "integer"}}"#;
let err = compile_json_schema_str(schema).expect_err("'if'/'then' should not be supported");
assert!(
matches!(err, JsonSchemaCompileError::UnsupportedKeyword(_)),
"expected UnsupportedKeyword, got: {err:?}"
);
}
#[test]
fn invalid_json_str() {
let err = compile_json_schema_str("not json at all }{").expect_err("invalid JSON must fail");
assert!(
matches!(err, JsonSchemaCompileError::InvalidJson(_)),
"expected InvalidJson, got: {err:?}"
);
}
#[test]
fn dangling_ref() {
let schema = r##"{"$ref": "#/$defs/Missing"}"##;
let err = compile_json_schema_str(schema).expect_err("dangling $ref must fail");
assert!(
matches!(err, JsonSchemaCompileError::DanglingRef(_)),
"expected DanglingRef, got: {err:?}"
);
}
#[test]
fn depth_exceeded() {
let mut schema = serde_json::json!({"type": "integer"});
for _ in 0..33 {
schema = serde_json::json!({"anyOf": [schema]});
}
let schema_str = serde_json::to_string(&schema).unwrap();
let err =
compile_json_schema_str(&schema_str).expect_err("depth > 32 must return DepthExceeded");
assert!(
matches!(err, JsonSchemaCompileError::DepthExceeded { limit: 32 }),
"expected DepthExceeded {{ limit: 32 }}, got: {err:?}"
);
}
#[test]
fn all_of_merges_objects() {
let schema = r#"{
"allOf": [
{
"type": "object",
"properties": {"x": {"type": "integer"}},
"required": ["x"]
},
{
"type": "object",
"properties": {"y": {"type": "string"}},
"required": ["y"]
}
]
}"#;
let g = compile_json_schema_str(schema).expect("allOf of objects must compile");
assert!(!g.rules.is_empty(), "merged object must produce rules");
let has_integer_nt = g.nt_names.values().any(|n| n.contains("integer"));
let has_string_nt = g.nt_names.values().any(|n| n.contains("string"));
assert!(has_integer_nt, "merged schema should contain integer NT");
assert!(has_string_nt, "merged schema should contain string NT");
}
#[test]
fn end_to_end_grammar_constraint_string_type() {
let schema = r#"{"type":"string"}"#;
let grammar = compile_json_schema_str(schema).expect("string schema must compile");
let c = GrammarConstraint::new(
grammar,
|id| if id < 128 { vec![id as u8] } else { vec![] },
128,
);
let mask = c
.allowed_tokens(&[], 128)
.expect("allowed_tokens must return Some");
assert!(
mask[b'"' as usize],
"quote character (0x22) must be allowed at start of string type"
);
for d in b'0'..=b'9' {
assert!(
!mask[d as usize],
"digit '{}' should NOT be allowed at start of string type",
d as char
);
}
assert!(!mask[b't' as usize], "'t' should not start a string");
assert!(!mask[b'f' as usize], "'f' should not start a string");
assert!(!mask[b'n' as usize], "'n' should not start a string");
}
#[test]
fn end_to_end_integer_constraint_advance() {
let schema = r#"{"type":"integer"}"#;
let mut c = ascii_constraint_from_schema(schema);
let mask = c.allowed_tokens(&[], 128).unwrap();
for d in b'0'..=b'9' {
assert!(
mask[d as usize],
"digit '{}' must be allowed at start",
d as char
);
}
assert!(
mask[b'-' as usize],
"'-' must be allowed at start of integer"
);
assert!(feed_str_constraint(&mut c, "123"));
assert!(c.is_complete(), "123 is a complete integer");
}
#[test]
fn compile_enum_bool_null() {
let schema = r#"{"enum":[true,false,null]}"#;
let mut rec_true = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec_true, "true"));
assert!(rec_true.is_accepting());
let mut rec_false = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec_false, "false"));
assert!(rec_false.is_accepting());
let mut rec_null = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec_null, "null"));
assert!(rec_null.is_accepting());
let mut rec_bad = recognizer_from_schema(schema);
let ok = feed_str_recognizer(&mut rec_bad, "maybe");
assert!(!ok || !rec_bad.is_accepting());
}
#[test]
fn object_empty_required() {
let schema = r#"{"type":"object","properties":{"x":{"type":"integer"}}}"#;
let mut rec = recognizer_from_schema(schema);
assert!(feed_str_recognizer(&mut rec, "{}"));
assert!(rec.is_accepting());
}
#[test]
fn compile_definitions_alias() {
let schema = r##"{
"definitions": {
"Num": {"type": "integer"}
},
"$ref": "#/definitions/Num"
}"##;
let g = compile_json_schema_str(schema).expect("'definitions' alias must compile");
assert!(!g.rules.is_empty());
let mut rec = EarleyRecognizer::new({
let mut g2 = g;
g2.normalise_terminals();
Arc::new(g2)
});
assert!(feed_str_recognizer(&mut rec, "7"));
assert!(rec.is_accepting());
}
#[test]
fn all_of_rejects_non_object() {
let schema = r#"{"allOf":[{"type":"string"},{"type":"integer"}]}"#;
let err = compile_json_schema_str(schema).expect_err("allOf of non-objects must fail");
assert!(
matches!(err, JsonSchemaCompileError::UnsupportedKeyword(_)),
"expected UnsupportedKeyword for allOf of non-objects, got: {err:?}"
);
}
#[test]
fn unsupported_keyword_pattern() {
let schema = r#"{"type":"string","pattern":"^[a-z]+"}"#;
let err = compile_json_schema_str(schema).expect_err("'pattern' should not be supported");
assert!(
matches!(err, JsonSchemaCompileError::UnsupportedKeyword(ref kw) if kw == "pattern"),
"expected UnsupportedKeyword(\"pattern\"), got: {err:?}"
);
}
#[test]
fn grammar_has_correct_start_nt() {
let g = compile_json_schema_str(r#"{"type":"boolean"}"#).expect("must compile");
let start = g.start();
let start_rules: Vec<_> = g.rules_for(start).collect();
assert!(
!start_rules.is_empty(),
"start NT {start} must have at least one rule, got 0"
);
}
#[test]
fn compile_json_schema_value_api() {
let schema_val = serde_json::json!({"type": "null"});
let g = compile_json_schema(&schema_val).expect("must compile from Value");
assert!(!g.rules.is_empty());
let mut rec = EarleyRecognizer::new({
let mut g2 = g;
g2.normalise_terminals();
Arc::new(g2)
});
assert!(feed_str_recognizer(&mut rec, "null"));
assert!(rec.is_accepting());
}