use crate::Atom;
use crate::Domain;
use crate::TreeValueReader;
use crate::ValueClass;
use crate::error::Error;
use crate::error::ExpectedKind;
use crate::error::SyntaxError;
use crate::hex;
use crate::CompoundClass;
use crate::Reader;
use crate::boundary as B;
use crate::reader::NextToken;
use crate::reader::ReaderResult;
use crate::source::BinarySource;
use lazy_static::lazy_static;
use num_bigint::BigInt;
use std::borrow::Cow;
use std::collections::VecDeque;
use std::convert::TryInto;
use std::default::Default;
use std::io;
use std::marker::PhantomData;
#[derive(Debug)]
enum Classification {
Atom(Atom<'static>),
Compound(CompoundClass),
Embedded,
Annotation,
EndCompound,
}
impl<'r> From<&'r Classification> for NextToken {
fn from(c: &'r Classification) -> Self {
match c {
Classification::Atom(a) => NextToken::Value(ValueClass::Atomic(a.into())),
Classification::Compound(c) => NextToken::Value(ValueClass::Compound(c.clone())),
Classification::Embedded => NextToken::Value(ValueClass::Embedded),
Classification::Annotation => NextToken::Annotation,
Classification::EndCompound => NextToken::End,
}
}
}
pub struct TextReader<'de, S: BinarySource<'de>> {
pub source: S,
classification_cache: VecDeque<Classification>,
phantom: PhantomData<&'de ()>,
}
impl<'de, S: BinarySource<'de>> TextReader<'de, S>
{
pub fn new(source: S) -> Self {
TextReader {
source,
classification_cache: Default::default(),
phantom: PhantomData,
}
}
fn peek(&mut self) -> io::Result<Option<u8>> {
self.source.peek()
}
#[inline(always)]
fn peek_noeof(&mut self) -> ReaderResult<u8> {
self.source.peek_noeof()
}
fn skip(&mut self) -> io::Result<()> {
self.source.skip()
}
#[inline(always)]
fn next_byte(&mut self) -> ReaderResult<u8> {
self.source.read()
}
fn skip_whitespace(&mut self) {
self.skip_whitespace_and_maybe_commas(false)
}
fn skip_whitespace_and_maybe_commas(&mut self, skip_commas: bool) {
while let Ok(Some(c)) = self.peek() {
match c {
b' ' | b'\t' | b'\r' | b'\n' => {
let _ = self.skip();
()
}
b',' if skip_commas => {
let _ = self.skip();
()
}
_ => break,
}
}
}
fn syntax_error(&mut self, message: &str) -> Error {
self.wrap_syntax_error(SyntaxError::Message(message.to_string()))
}
fn decode_utf8(&mut self, bs: Vec<u8>) -> ReaderResult<String> {
String::from_utf8(bs).map_err(|_| self.syntax_error("Invalid UTF-8"))
}
fn comment_line(&mut self) -> ReaderResult<String> {
let mut bs = Vec::new();
loop {
match self.next_byte()? {
b'\r' | b'\n' => return Ok(self.decode_utf8(bs)?),
b => bs.push(b),
}
}
}
fn read_hex_float(&mut self) -> ReaderResult<Atom<'static>> {
if self.next_byte()? != b'"' {
return Err(self.syntax_error("Missing open-double-quote in hex-encoded floating-point number"));
}
let bs = self.read_hex_binary()?;
if bs.len() != 8 {
return Err(self.syntax_error("Incorrect number of bytes in hex-encoded floating-point number"));
}
Ok(Atom::Double(f64::from_bits(u64::from_be_bytes(bs.try_into().unwrap()))))
}
fn read_stringlike<X, H, R>(
&mut self,
mut seed: R,
xform_item: X,
terminator: u8,
hexescape: u8,
hexescaper: H,
) -> ReaderResult<R>
where
X: Fn(&mut Self, &mut R, u8) -> ReaderResult<()>,
H: Fn(&mut Self, &mut R) -> ReaderResult<()>,
{
loop {
match self.next_byte()? {
c if c == terminator => return Ok(seed),
b'\\' => match self.next_byte()? {
c if c == hexescape => hexescaper(self, &mut seed)?,
c if c == terminator || c == b'\\' || c == b'/' => xform_item(self, &mut seed, c)?,
b'b' => xform_item(self, &mut seed, b'\x08')?,
b'f' => xform_item(self, &mut seed, b'\x0c')?,
b'n' => xform_item(self, &mut seed, b'\x0a')?,
b'r' => xform_item(self, &mut seed, b'\x0d')?,
b't' => xform_item(self, &mut seed, b'\x09')?,
_ => return Err(self.syntax_error("Invalid escape code")),
},
c => xform_item(self, &mut seed, c)?,
}
}
}
fn hexnum(&mut self, count: usize) -> ReaderResult<u32> {
let mut v: u32 = 0;
for _ in 0 .. count {
let c = self.next_byte()?;
match (c as char).to_digit(16) {
Some(d) =>
v = v << 4 | d,
None =>
return Err(self.syntax_error("Bad hex escape")),
}
}
Ok(v)
}
fn append_codepoint(&mut self, bs: &mut Vec<u8>, n: u32) -> ReaderResult<()> {
let c = char::from_u32(n).ok_or_else(|| self.syntax_error("Bad code point"))?;
let mut buf = [0; 4];
let _ = c.encode_utf8(&mut buf);
bs.extend(&buf[0 .. c.len_utf8()]);
Ok(())
}
fn read_string(&mut self, delimiter: u8) -> ReaderResult<String> {
let raw = self.read_stringlike(
Vec::new(),
|_r, bs, c| Ok(bs.push(c)),
delimiter,
b'u',
|r, bs| {
let n1 = r.hexnum(4)?;
if (0xd800 ..= 0xdbff).contains(&n1) {
let mut ok = true;
ok = ok && r.next_byte()? == b'\\';
ok = ok && r.next_byte()? == b'u';
if !ok {
Err(r.syntax_error("Missing second half of surrogate pair"))
} else {
let n2 = r.hexnum(4)?;
if (0xdc00 ..= 0xdfff).contains(&n2) {
let n = ((n1 - 0xd800) << 10) + (n2 - 0xdc00) + 0x10000;
r.append_codepoint(bs, n)
} else {
Err(r.syntax_error("Bad second half of surrogate pair"))
}
}
} else {
r.append_codepoint(bs, n1)
}
})?;
self.decode_utf8(raw)
}
fn read_literal_binary(&mut self) -> ReaderResult<Atom<'static>> {
Ok(Atom::ByteString(Cow::Owned(self.read_stringlike(
Vec::new(),
|_r, bs, b| Ok(bs.push(b)),
b'"',
b'x',
|r, bs| Ok(bs.push(r.hexnum(2)? as u8)))?)))
}
fn read_hex_binary(&mut self) -> ReaderResult<Vec<u8>> {
let mut s = String::new();
loop {
self.skip_whitespace();
let c1 = self.next_byte()? as char;
if c1 == '"' {
return Ok(hex::HexParser::Strict.decode(&s).unwrap());
}
let c2 = self.next_byte()? as char;
if !(c1.is_digit(16) && c2.is_digit(16)) {
return Err(self.syntax_error("Invalid hex binary"));
}
s.push(c1);
s.push(c2);
}
}
fn read_base64_binary(&mut self) -> ReaderResult<Atom<'static>> {
let mut bs = Vec::new();
loop {
self.skip_whitespace();
let mut c = self.next_byte()?;
if c == b']' {
let bs = base64::decode_config(&self.decode_utf8(bs)?, base64::STANDARD_NO_PAD)
.map_err(|_| self.syntax_error("Invalid base64 character"))?;
return Ok(Atom::ByteString(Cow::Owned(bs)));
}
if c == b'-' { c = b'+'; }
if c == b'_' { c = b'/'; }
if c == b'=' { continue; }
bs.push(c);
}
}
fn require_delimiter(&mut self, msg: &'static str) -> ReaderResult<()> {
if self.delimiter_follows()? {
Ok(())
} else {
Err(self.syntax_error(msg))
}
}
fn delimiter_follows(&mut self) -> io::Result<bool> {
let c = match self.peek()? {
None => return Ok(true),
Some(c) if (c as char).is_whitespace() => return Ok(true),
Some(c) => c,
};
Ok(match c {
b'(' | b')' | b'{' | b'}' | b'[' | b']' | b'<' | b'>' | b'"' | b'\'' | b';' | b','
| b'@' | b'#' | b':' | b' ' => true,
_ => false,
})
}
fn read_raw_symbol_or_number(&mut self, mut bs: Vec<u8>) -> ReaderResult<Atom<'static>> {
lazy_static! {
static ref NUMBER_RE: regex::Regex =
regex::Regex::new(r"^([-+]?\d+)((\.\d+([eE][-+]?\d+)?)|([eE][-+]?\d+))?$")
.unwrap();
}
while !self.delimiter_follows()? {
bs.push(self.next_byte()?);
}
let s = self.decode_utf8(bs)?;
match NUMBER_RE.captures(&s) {
None => Ok(Atom::symbol(s)),
Some(m) => match m.get(2) {
None => Ok(Atom::SignedInteger(Cow::Owned(s.parse::<BigInt>().map_err(
|_| self.syntax_error(&format!(
"Invalid signed-integer number: {:?}", s)))?.into()))),
Some(_) => Ok(Atom::Double(s.parse::<f64>().map_err(
|_| self.syntax_error(&format!(
"Invalid double-precision floating-point number: {:?}", s)))?)),
},
}
}
fn prime_classification_cache(&mut self) -> ReaderResult<()> {
if !self.classification_cache.is_empty() {
return Ok(());
}
self.skip_whitespace();
let c = match self.peek()? {
None => return Ok(()),
Some(c) => c,
};
self.skip()?;
let cl = match c {
b'"' => Classification::Atom(Atom::String(Cow::Owned(self.read_string(b'"')?))),
b'\'' => Classification::Atom(Atom::Symbol(Cow::Owned(self.read_string(b'\'')?))),
b':' => Err(self.syntax_error("Unexpected key/value separator between items"))?,
b';' => Err(self.syntax_error("Semicolon is reserved syntax"))?,
b'@' => Classification::Annotation,
b'#' => match self.peek_noeof()? {
b'\n' | b'\r' => {
let line = Atom::String(Cow::Owned(self.comment_line()?));
self.classification_cache.push_back(Classification::Annotation);
self.classification_cache.push_back(Classification::Atom(line));
return Ok(());
}
other => {
self.skip()?;
match other {
b' ' | b'\t' => {
let line = Atom::String(Cow::Owned(self.comment_line()?));
self.classification_cache.push_back(Classification::Annotation);
self.classification_cache.push_back(Classification::Atom(line));
return Ok(());
}
b'!' => {
let line = Atom::String(Cow::Owned(self.comment_line()?));
self.classification_cache.push_back(Classification::Annotation);
self.classification_cache.push_back(
Classification::Compound(CompoundClass::Record));
self.classification_cache.push_back(
Classification::Atom(Atom::Symbol(Cow::Borrowed("interpreter"))));
self.classification_cache.push_back(Classification::Atom(line));
self.classification_cache.push_back(Classification::EndCompound);
return Ok(());
}
b'f' => {
self.require_delimiter("Delimiter must follow #f")?;
Classification::Atom(Atom::Boolean(false))
}
b't' => {
self.require_delimiter("Delimiter must follow #t")?;
Classification::Atom(Atom::Boolean(true))
}
b'{' => Classification::Compound(CompoundClass::Set),
b'"' => Classification::Atom(self.read_literal_binary()?),
b'x' => match self.next_byte()? {
b'"' => Classification::Atom(Atom::ByteString(self.read_hex_binary()?.into())),
b'd' => Classification::Atom(self.read_hex_float()?),
_ => Err(self.syntax_error("Invalid #x syntax"))?,
},
b'[' => Classification::Atom(self.read_base64_binary()?),
b':' => Classification::Embedded,
other => Err(self.syntax_error(&format!("Invalid # syntax: {:?}", other)))?,
}
}
}
b'<' => Classification::Compound(CompoundClass::Record),
b'[' => Classification::Compound(CompoundClass::Sequence),
b'{' => Classification::Compound(CompoundClass::Dictionary),
b'>' => Classification::EndCompound,
b']' => Classification::EndCompound,
b'}' => Classification::EndCompound,
b',' => Err(self.syntax_error("Unexpected ,"))?,
other => Classification::Atom(self.read_raw_symbol_or_number(vec![other])?),
};
self.classification_cache.push_back(cl);
Ok(())
}
}
impl<'de, S: BinarySource<'de>> Reader<'de> for TextReader<'de, S>
{
type ValueReader<D: Domain> = TreeValueReader;
type IOValueReader = TreeValueReader;
fn peek_class(&mut self) -> ReaderResult<Option<NextToken>> {
self.prime_classification_cache()?;
Ok(self.classification_cache.front().map(|a| a.into()))
}
fn wrap_syntax_error(&mut self, e: SyntaxError) -> Error {
self.source.wrap_syntax_error(e)
}
fn skip_atom(&mut self) -> ReaderResult<()> {
let _ = self.next_atom()?;
Ok(())
}
fn next_atom(&mut self) -> ReaderResult<Atom<'de>> {
self.skip_annotations()?;
self.prime_classification_cache()?;
match self.classification_cache.pop_front() {
None => Err(self.wrap_syntax_error(SyntaxError::eof())),
Some(a) => match a {
Classification::Atom(a) => Ok(a),
Classification::Compound(_) => Err(self.syntax_error("Unexpected compound value"))?,
Classification::Embedded => Err(self.syntax_error("Unexpected embedded value"))?,
Classification::Annotation =>
unreachable!("Annotations are supposed to have been skipped already"),
Classification::EndCompound =>
Err(self.syntax_error("Unexpected end-compound token"))?,
}
}
}
fn open_record(&mut self) -> ReaderResult<()> {
self.skip_annotations()?;
if self.peek_class()? != Some(NextToken::Value(ValueClass::Compound(CompoundClass::Record))) {
return Err(self.wrap_syntax_error(SyntaxError::Expected(ExpectedKind::Record)));
}
self.classification_cache.pop_front();
Ok(())
}
fn open_sequence(&mut self) -> ReaderResult<()> {
self.skip_annotations()?;
if self.peek_class()? != Some(NextToken::Value(ValueClass::Compound(CompoundClass::Sequence))) {
return Err(self.wrap_syntax_error(SyntaxError::Expected(ExpectedKind::Sequence)));
}
self.classification_cache.pop_front();
Ok(())
}
fn open_set(&mut self) -> ReaderResult<()> {
self.skip_annotations()?;
if self.peek_class()? != Some(NextToken::Value(ValueClass::Compound(CompoundClass::Set))) {
return Err(self.wrap_syntax_error(SyntaxError::Expected(ExpectedKind::Set)));
}
self.classification_cache.pop_front();
Ok(())
}
fn open_dictionary(&mut self) -> ReaderResult<()> {
self.skip_annotations()?;
if self.peek_class()? != Some(NextToken::Value(ValueClass::Compound(CompoundClass::Dictionary))) {
return Err(self.wrap_syntax_error(SyntaxError::Expected(ExpectedKind::Dictionary)));
}
self.classification_cache.pop_front();
Ok(())
}
#[inline]
fn boundary(&mut self, b: &B::Type) -> ReaderResult<()> {
match b {
B::Type {
closing: Some(B::Item::DictionaryKey),
opening: Some(B::Item::DictionaryValue),
} => {
self.skip_whitespace();
if self.next_byte()? != b':' {
Err(self.syntax_error("Missing expected key/value separator"))?;
}
},
| B::Type {closing: Some(B::Item::DictionaryValue), ..}
| B::Type {opening: Some(B::Item::DictionaryKey), ..}
| B::Type {closing: Some(B::Item::SequenceValue), ..}
| B::Type {opening: Some(B::Item::SequenceValue), ..}
| B::Type {closing: Some(B::Item::SetValue), ..}
| B::Type {opening: Some(B::Item::SetValue), ..}
=> {
self.skip_whitespace_and_maybe_commas(true);
}
_ => (),
}
Ok(())
}
fn close_compound(&mut self, b: &mut B::Type, i: &B::Item) -> ReaderResult<bool> {
if self.classification_cache.is_empty() {
self.skip_whitespace_and_maybe_commas(!matches!(i, B::Item::RecordField));
match self.peek_noeof()? {
b'>' | b']' | b'}' => {
self.skip()?;
return Ok(true);
}
_ => (),
}
} else {
if let Some(Classification::EndCompound) = self.classification_cache.front() {
self.classification_cache.pop_front();
return Ok(true);
}
}
b.shift(Some(i.clone()));
self.boundary(b)?;
Ok(false)
}
fn open_embedded(&mut self) -> ReaderResult<()> {
self.skip_annotations()?;
if self.peek_class()? != Some(NextToken::Value(ValueClass::Embedded)) {
return Err(self.wrap_syntax_error(SyntaxError::Expected(ExpectedKind::Embedded)));
}
self.classification_cache.pop_front();
Ok(())
}
fn close_embedded(&mut self) -> ReaderResult<()> {
Ok(())
}
fn mark(&mut self) -> io::Result<usize> {
if !self.classification_cache.is_empty() {
panic!("Cannot mark with nonempty classification_cache");
}
self.source.mark()
}
fn restore(&mut self, mark: usize) -> io::Result<()> {
self.classification_cache.clear();
self.source.restore(mark)
}
fn open_annotation(&mut self) -> ReaderResult<()> {
let _ = self.peek_class()?;
match self.classification_cache.pop_front() {
None =>
unreachable!("peek_class should have primed the cache"),
Some(Classification::Annotation) =>
Ok(()),
Some(_) =>
Err(self.wrap_syntax_error(SyntaxError::Expected(ExpectedKind::Annotation))),
}
}
fn close_annotation(&mut self) -> ReaderResult<()> {
Ok(())
}
}