use fig_core::kit::xml as core_xml;
use crate::Span;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum TokenKind {
Lt,
LtSlash,
Gt,
SlashGt,
Eq,
Name,
AttrValue,
CharData,
CData,
Eof,
}
impl TokenKind {
fn from_core(k: core_xml::Kind) -> TokenKind {
match k {
core_xml::Kind::Lt => TokenKind::Lt,
core_xml::Kind::LtSlash => TokenKind::LtSlash,
core_xml::Kind::Gt => TokenKind::Gt,
core_xml::Kind::SlashGt => TokenKind::SlashGt,
core_xml::Kind::Eq => TokenKind::Eq,
core_xml::Kind::Name => TokenKind::Name,
core_xml::Kind::AttrValue => TokenKind::AttrValue,
core_xml::Kind::CharData => TokenKind::CharData,
core_xml::Kind::Cdata => TokenKind::CData,
core_xml::Kind::Eof => TokenKind::Eof,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum ErrorKind {
UnclosedTag,
UnclosedAttributeValue,
UnclosedComment,
UnclosedCData,
UnclosedPi,
UnsupportedDoctype,
InvalidMarkup,
UnexpectedCharacter,
OutOfMemory,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Error {
pub kind: ErrorKind,
pub at: usize,
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let what = match self.kind {
ErrorKind::UnclosedTag => "a tag is not closed",
ErrorKind::UnclosedAttributeValue => "an attribute value has no closing quote",
ErrorKind::UnclosedComment => "a comment has no `-->`",
ErrorKind::UnclosedCData => "a CDATA section has no `]]>`",
ErrorKind::UnclosedPi => "a processing instruction has no `?>`",
ErrorKind::UnsupportedDoctype => "a DOCTYPE with an internal subset is not supported",
ErrorKind::InvalidMarkup => "markup is not well-formed here",
ErrorKind::UnexpectedCharacter => "unexpected character inside a tag",
ErrorKind::OutOfMemory => "out of memory",
};
write!(f, "{what} (at byte {})", self.at)
}
}
impl std::error::Error for Error {}
pub fn tokenize(input: &[u8]) -> Result<Vec<Token>, Error> {
if input.len() > u32::MAX as usize {
return Err(Error {
kind: ErrorKind::OutOfMemory,
at: 0,
});
}
let mut tz = core_xml::Tokenizer::new(input);
match tz.tokenize() {
Ok(tokens) => Ok(tokens
.iter()
.map(|t| Token {
kind: TokenKind::from_core(t.kind),
span: Span {
start: t.span.start,
end: t.span.end,
},
})
.collect()),
Err(e) => {
let kind = match e.name() {
"UnclosedTag" => ErrorKind::UnclosedTag,
"UnclosedAttributeValue" => ErrorKind::UnclosedAttributeValue,
"UnclosedComment" => ErrorKind::UnclosedComment,
"UnclosedCData" => ErrorKind::UnclosedCData,
"UnclosedPI" => ErrorKind::UnclosedPi,
"UnsupportedDoctype" => ErrorKind::UnsupportedDoctype,
"InvalidMarkup" => ErrorKind::InvalidMarkup,
"UnexpectedCharacter" => ErrorKind::UnexpectedCharacter,
_ => ErrorKind::OutOfMemory,
};
Err(Error { kind, at: tz.pos })
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tokens_and_refusals() {
let t = tokenize(b"<a>x</a>").unwrap();
assert_eq!(t.len(), 8);
assert_eq!(t[3].kind, TokenKind::CharData);
assert_eq!(t[3].span, Span { start: 3, end: 4 });
assert_eq!(
tokenize(b"<a").unwrap_err(),
Error {
kind: ErrorKind::UnclosedTag,
at: 2
}
);
}
}