use std::path::Path;
#[derive(Debug)]
pub enum XmlGrammarCodegenError {
ReadSource(String, std::io::Error),
ProductionNotFound(&'static str),
UnknownRhsToken {
production: &'static str,
token: String,
},
}
impl core::fmt::Display for XmlGrammarCodegenError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::ReadSource(p, e) => write!(f, "read source {p}: {e}"),
Self::ProductionNotFound(name) => {
write!(f, "production `{name}` not found in W3C XML 1.0 spec")
}
Self::UnknownRhsToken { production, token } => {
write!(
f,
"production `{production}` RHS contained unrecognised token `{token}`"
)
}
}
}
}
impl std::error::Error for XmlGrammarCodegenError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CodePointRange {
pub lo: u32,
pub hi: u32,
}
pub fn generate_xml_grammar_source(spec_path: &Path) -> Result<String, XmlGrammarCodegenError> {
let spec_bytes = std::fs::read_to_string(spec_path)
.map_err(|e| XmlGrammarCodegenError::ReadSource(spec_path.display().to_string(), e))?;
generate_xml_grammar_from_source(&spec_bytes)
}
pub fn generate_xml_grammar_from_source(
spec_bytes: &str,
) -> Result<String, XmlGrammarCodegenError> {
let char_ranges = extract_production(spec_bytes, "Char")?;
let name_start_char_ranges = extract_production(spec_bytes, "NameStartChar")?;
let name_char_ranges = extract_production(spec_bytes, "NameChar")?;
let predefined_entities = extract_predefined_entities(spec_bytes)?;
let mut out = String::new();
out.push_str(
"// Generated by pr4xis::codegen::xml_grammar from the loaded W3C XML 1.0\n\
// Fifth Edition spec (Bray et al. 2008). Do not edit by hand — change the\n\
// upstream spec source `xml_1_0_fifth_edition@2008` and re-run cargo build.\n\
//\n\
// Each table corresponds to one EBNF production from §2.2 or §2.3 of the\n\
// spec, or to the §4.6 predefined-entities declarations. Inclusive\n\
// ranges; lo ≤ hi; sorted lo-ascending in source order.\n\n",
);
emit_range_table(&mut out, "CHAR_RANGES", "§2.2 \\[2\\] Char", &char_ranges);
emit_predicate(&mut out, "is_char", "CHAR_RANGES");
emit_range_table(
&mut out,
"NAME_START_CHAR_RANGES",
"§2.3 \\[4\\] NameStartChar",
&name_start_char_ranges,
);
emit_predicate(&mut out, "is_name_start_char", "NAME_START_CHAR_RANGES");
emit_range_table(
&mut out,
"NAME_CHAR_RANGES",
"§2.3 \\[4a\\] NameChar",
&name_char_ranges,
);
emit_predicate(&mut out, "is_name_char", "NAME_CHAR_RANGES");
emit_predefined_entities(&mut out, &predefined_entities);
Ok(out)
}
fn extract_production(
spec_bytes: &str,
lhs_name: &'static str,
) -> Result<Vec<CodePointRange>, XmlGrammarCodegenError> {
let rhs = locate_rhs(spec_bytes, lhs_name)
.ok_or(XmlGrammarCodegenError::ProductionNotFound(lhs_name))?;
parse_rhs_ranges(rhs, spec_bytes, lhs_name)
}
fn locate_rhs<'a>(spec_bytes: &'a str, lhs_name: &str) -> Option<&'a str> {
let mut cursor = 0;
while cursor < spec_bytes.len() {
let rest = &spec_bytes[cursor..];
let lhs_open = rest.find("<lhs")?;
let after_open = &rest[lhs_open + 4..]; let gt = after_open.find('>')?;
let lhs_content_start = lhs_open + 4 + gt + 1;
let after_content = &rest[lhs_content_start..];
let lhs_close = after_content.find("</lhs>")?;
let content = &after_content[..lhs_close];
if content.trim() == lhs_name {
let after_lhs = &after_content[lhs_close + "</lhs>".len()..];
let rhs_open = after_lhs.find("<rhs")?;
let after_rhs_open = &after_lhs[rhs_open + 4..];
let gt2 = after_rhs_open.find('>')?;
let rhs_content_start = rhs_open + 4 + gt2 + 1;
let after_rhs_content = &after_lhs[rhs_content_start..];
let rhs_close = after_rhs_content.find("</rhs>")?;
return Some(&after_rhs_content[..rhs_close]);
}
cursor += lhs_open + 4 + gt + 1 + lhs_close + "</lhs>".len();
}
None
}
fn parse_rhs_ranges(
rhs: &str,
spec_bytes: &str,
production: &'static str,
) -> Result<Vec<CodePointRange>, XmlGrammarCodegenError> {
let mut ranges = Vec::new();
for raw in rhs.split('|') {
let tok = raw.trim();
if tok.is_empty() {
continue;
}
if let Some(nt_ref) = extract_nt_reference(tok) {
let inner_rhs = locate_rhs(spec_bytes, nt_ref)
.ok_or(XmlGrammarCodegenError::ProductionNotFound(production))?;
let mut inner = parse_rhs_ranges_owned(inner_rhs, spec_bytes, production)?;
ranges.append(&mut inner);
continue;
}
let range = parse_token(tok).ok_or_else(|| XmlGrammarCodegenError::UnknownRhsToken {
production,
token: tok.to_string(),
})?;
ranges.push(range);
}
Ok(ranges)
}
fn parse_rhs_ranges_owned(
rhs: &str,
spec_bytes: &str,
production: &'static str,
) -> Result<Vec<CodePointRange>, XmlGrammarCodegenError> {
parse_rhs_ranges(rhs, spec_bytes, production)
}
fn extract_nt_reference(tok: &str) -> Option<&str> {
let after_open = tok.strip_prefix("<nt ")?;
let gt = after_open.find('>')?;
let after_content_start = &after_open[gt + 1..];
let nt_close = after_content_start.find("</nt>")?;
let content = &after_content_start[..nt_close];
Some(content.trim())
}
fn parse_token(tok: &str) -> Option<CodePointRange> {
if let Some(inner) = tok.strip_prefix("[#x").and_then(|s| s.strip_suffix(']')) {
let (lo_hex, hi_hex) = inner.split_once("-#x")?;
let lo = u32::from_str_radix(lo_hex.trim(), 16).ok()?;
let hi = u32::from_str_radix(hi_hex.trim(), 16).ok()?;
return Some(CodePointRange { lo, hi });
}
if let Some(hex) = tok.strip_prefix("#x") {
let cp = u32::from_str_radix(hex.trim(), 16).ok()?;
return Some(CodePointRange { lo: cp, hi: cp });
}
if let Some(inner) = tok.strip_prefix('[').and_then(|s| s.strip_suffix(']')) {
let chars: Vec<char> = inner.chars().collect();
if chars.len() == 3 && chars[1] == '-' {
return Some(CodePointRange {
lo: chars[0] as u32,
hi: chars[2] as u32,
});
}
}
for delim in ['"', '\''] {
if let Some(inner) = tok.strip_prefix(delim).and_then(|s| s.strip_suffix(delim)) {
let chars: Vec<char> = inner.chars().collect();
if chars.len() == 1 {
let cp = chars[0] as u32;
return Some(CodePointRange { lo: cp, hi: cp });
}
}
}
None
}
fn emit_range_table(out: &mut String, name: &str, citation: &str, ranges: &[CodePointRange]) {
out.push_str(&format!(
"/// W3C XML 1.0 Fifth Edition {citation}.\n\
/// {n} inclusive code-point ranges, derived from the loaded spec.\n\
#[allow(dead_code)]\n\
pub const {name}: &[(u32, u32)] = &[\n",
citation = citation,
n = ranges.len(),
name = name,
));
for r in ranges {
out.push_str(&format!(" (0x{:X}, 0x{:X}),\n", r.lo, r.hi));
}
out.push_str("];\n\n");
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PredefinedEntityDecl {
pub name: String,
pub replacement: char,
pub replacement_literal: String,
}
fn extract_predefined_entities(
spec_bytes: &str,
) -> Result<Vec<PredefinedEntityDecl>, XmlGrammarCodegenError> {
let section_anchor = spec_bytes.find(r#"id="sec-predefined-ent""#).ok_or(
XmlGrammarCodegenError::ProductionNotFound("sec-predefined-ent"),
)?;
let after_anchor = &spec_bytes[section_anchor..];
let eg_open = after_anchor
.find("<eg>")
.ok_or(XmlGrammarCodegenError::ProductionNotFound(
"sec-predefined-ent/eg",
))?;
let after_eg_open = &after_anchor[eg_open + "<eg>".len()..];
let eg_close =
after_eg_open
.find("</eg>")
.ok_or(XmlGrammarCodegenError::ProductionNotFound(
"sec-predefined-ent/eg-close",
))?;
let eg_body = &after_eg_open[..eg_close];
let cdata = eg_body
.trim()
.strip_prefix("<![CDATA[")
.and_then(|s| s.strip_suffix("]]>"))
.ok_or(XmlGrammarCodegenError::ProductionNotFound(
"sec-predefined-ent/cdata",
))?;
let mut out = Vec::new();
for raw_line in cdata.lines() {
let line = raw_line.trim();
if line.is_empty() {
continue;
}
let decl = parse_predefined_entity_line(line).ok_or_else(|| {
XmlGrammarCodegenError::UnknownRhsToken {
production: "PredefinedEntity",
token: line.to_string(),
}
})?;
out.push(decl);
}
Ok(out)
}
fn parse_predefined_entity_line(line: &str) -> Option<PredefinedEntityDecl> {
let rest = line.strip_prefix("<!ENTITY")?.trim_start();
let space = rest.find(char::is_whitespace)?;
let name = rest[..space].trim().to_string();
let after_name = rest[space..].trim_start();
let quote_char = after_name.chars().next()?;
if quote_char != '"' && quote_char != '\'' {
return None;
}
let after_quote = &after_name[quote_char.len_utf8()..];
let close_quote = after_quote.find(quote_char)?;
let literal = &after_quote[..close_quote];
let after_value = after_quote[close_quote + quote_char.len_utf8()..].trim_start();
if !after_value.starts_with('>') {
return None;
}
let replacement = resolve_predefined_entity_value(literal)?;
Some(PredefinedEntityDecl {
name,
replacement,
replacement_literal: literal.to_string(),
})
}
fn resolve_predefined_entity_value(literal: &str) -> Option<char> {
let trimmed = literal.trim();
let last_hash = trimmed.rfind('#')?;
let after_hash = &trimmed[last_hash + 1..];
let digits = after_hash.strip_suffix(';')?;
let cp = if let Some(hex) = digits.strip_prefix('x') {
u32::from_str_radix(hex, 16).ok()?
} else {
digits.parse::<u32>().ok()?
};
char::from_u32(cp)
}
fn emit_predefined_entities(out: &mut String, entities: &[PredefinedEntityDecl]) {
let mut sorted: Vec<&PredefinedEntityDecl> = entities.iter().collect();
sorted.sort_by(|a, b| a.name.cmp(&b.name));
out.push_str(
"/// W3C XML 1.0 Fifth Edition §4.6 predefined-entities table.\n\
/// `(name, replacement_char)` projected from the §4.6 `<!ENTITY …>`\n\
/// declarations in the loaded spec source. The parser uses this\n\
/// table to resolve `&name;` references for the five spec-mandated\n\
/// entities (per `feedback_bottom_up_loaded_not_encoded`).\n\
#[allow(dead_code)]\n\
pub const PREDEFINED_ENTITIES: &[(&str, char)] = &[\n",
);
for d in &sorted {
out.push_str(&format!(
" // <!ENTITY {name:<4} {literal:?}> → U+{cp:04X}\n (\"{name}\", '{esc}'),\n",
name = d.name,
literal = d.replacement_literal,
cp = d.replacement as u32,
esc = escape_char_literal(d.replacement),
));
}
out.push_str("];\n\n");
out.push_str(
"/// True iff `name` is one of the five §4.6 predefined-entity\n\
/// names. Returns the resolved replacement character.\n\
#[must_use]\n\
#[allow(dead_code)]\n\
pub fn resolve_predefined_entity(name: &str) -> Option<char> {\n\
\x20 PREDEFINED_ENTITIES\n\
\x20 .iter()\n\
\x20 .find(|(n, _)| *n == name)\n\
\x20 .map(|(_, c)| *c)\n\
}\n\n",
);
}
fn escape_char_literal(ch: char) -> String {
match ch {
'\\' => "\\\\".into(),
'\'' => "\\'".into(),
'"' => "\\\"".into(),
'\t' => "\\t".into(),
'\n' => "\\n".into(),
'\r' => "\\r".into(),
c if c.is_ascii_graphic() || c == ' ' => c.to_string(),
c => format!("\\u{{{:X}}}", c as u32),
}
}
fn emit_predicate(out: &mut String, fname: &str, table: &str) {
out.push_str(&format!(
"/// True iff `c` is in the {table} table.\n\
#[must_use]\n\
#[allow(dead_code)]\n\
pub fn {fname}(c: u32) -> bool {{\n\
\x20 {table}.iter().any(|&(lo, hi)| c >= lo && c <= hi)\n\
}}\n\n",
fname = fname,
table = table,
));
}
#[cfg(test)]
mod tests {
use super::*;
#[crate::praxis_value(Verifiable)]
#[test]
fn parses_hex_range() {
let r = parse_token("[#x20-#xD7FF]").unwrap();
assert_eq!(r.lo, 0x20);
assert_eq!(r.hi, 0xD7FF);
}
#[crate::praxis_value(Verifiable)]
#[test]
fn parses_single_hex() {
let r = parse_token("#x9").unwrap();
assert_eq!(r.lo, 9);
assert_eq!(r.hi, 9);
}
#[crate::praxis_value(Verifiable)]
#[test]
fn parses_ascii_range() {
let r = parse_token("[A-Z]").unwrap();
assert_eq!(r.lo, 0x41);
assert_eq!(r.hi, 0x5A);
}
#[crate::praxis_value(Verifiable)]
#[test]
fn parses_ascii_literal() {
let r = parse_token("\":\"").unwrap();
assert_eq!(r.lo, 0x3A);
assert_eq!(r.hi, 0x3A);
let r = parse_token("'_'").unwrap();
assert_eq!(r.lo, 0x5F);
assert_eq!(r.hi, 0x5F);
}
#[crate::praxis_value(Honest)]
#[test]
fn rejects_unknown_token() {
assert!(parse_token("xyzzy").is_none());
assert!(parse_token("[abc]").is_none()); }
#[crate::praxis_value(Verifiable)]
#[test]
fn parses_full_char_rhs() {
let rhs = "#x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]";
let ranges = parse_rhs_ranges(rhs, "", "Char").unwrap();
assert_eq!(ranges.len(), 6);
assert_eq!(ranges[0], CodePointRange { lo: 9, hi: 9 });
assert_eq!(
ranges[3],
CodePointRange {
lo: 0x20,
hi: 0xD7FF
}
);
assert_eq!(
ranges[5],
CodePointRange {
lo: 0x10000,
hi: 0x10FFFF
}
);
}
#[crate::praxis_value(Verifiable)]
#[test]
fn extracts_nt_reference() {
assert_eq!(
extract_nt_reference("<nt def=\"NT-NameStartChar\">NameStartChar</nt>"),
Some("NameStartChar")
);
assert_eq!(extract_nt_reference("[A-Z]"), None);
assert_eq!(extract_nt_reference("#x20"), None);
}
#[crate::praxis_value(Verifiable)]
#[test]
fn locate_rhs_tolerates_attributes_on_lhs_and_rhs() {
let spec = "<lhs diff=\"chg\">NameStartChar</lhs>\
<rhs diff=\"chg\">\":\" | [A-Z]</rhs>";
let rhs = locate_rhs(spec, "NameStartChar").unwrap();
assert!(rhs.contains("[A-Z]"));
}
#[crate::praxis_value(Verifiable)]
#[test]
fn resolves_predefined_entity_double_escape() {
assert_eq!(resolve_predefined_entity_value("&#60;"), Some('<'));
assert_eq!(resolve_predefined_entity_value("&#38;"), Some('&'));
}
#[crate::praxis_value(Verifiable)]
#[test]
fn resolves_predefined_entity_single_escape() {
assert_eq!(resolve_predefined_entity_value(">"), Some('>'));
assert_eq!(resolve_predefined_entity_value("'"), Some('\''));
assert_eq!(resolve_predefined_entity_value("""), Some('"'));
}
#[crate::praxis_value(Verifiable)]
#[test]
fn parses_predefined_entity_line_with_extra_whitespace() {
let decl = parse_predefined_entity_line(r#"<!ENTITY lt "&#60;">"#).unwrap();
assert_eq!(decl.name, "lt");
assert_eq!(decl.replacement, '<');
assert_eq!(decl.replacement_literal, "&#60;");
}
#[crate::praxis_value(Verifiable)]
#[test]
fn extracts_all_five_predefined_entities_from_spec() {
let spec = r#"<div2 id="sec-predefined-ent">
<eg><![CDATA[<!ENTITY lt "&#60;">
<!ENTITY gt ">">
<!ENTITY amp "&#38;">
<!ENTITY apos "'">
<!ENTITY quot """>]]></eg>
</div2>"#;
let entities = extract_predefined_entities(spec).unwrap();
assert_eq!(entities.len(), 5);
let by_name: std::collections::HashMap<&str, char> = entities
.iter()
.map(|e| (e.name.as_str(), e.replacement))
.collect();
assert_eq!(by_name.get("lt"), Some(&'<'));
assert_eq!(by_name.get("gt"), Some(&'>'));
assert_eq!(by_name.get("amp"), Some(&'&'));
assert_eq!(by_name.get("apos"), Some(&'\''));
assert_eq!(by_name.get("quot"), Some(&'"'));
}
#[crate::praxis_value(Verifiable)]
#[test]
fn name_char_resolves_via_nt_expansion() {
let spec = "<lhs>NameStartChar</lhs>\
<rhs>\":\" | [A-Z]</rhs>\
<lhs>NameChar</lhs>\
<rhs><nt def=\"NT-NameStartChar\">NameStartChar</nt> | \"-\" | [0-9]</rhs>";
let ranges = extract_production(spec, "NameChar").unwrap();
assert_eq!(ranges.len(), 4);
assert_eq!(ranges[0], CodePointRange { lo: 0x3A, hi: 0x3A }); assert_eq!(ranges[1], CodePointRange { lo: 0x41, hi: 0x5A }); assert_eq!(ranges[2], CodePointRange { lo: 0x2D, hi: 0x2D }); assert_eq!(ranges[3], CodePointRange { lo: 0x30, hi: 0x39 }); }
}