#![allow(dead_code)]
use std::collections::BTreeMap;
use std::fs;
use std::path::PathBuf;
use token_align::{get_alignments, get_alignments_with, get_charmap, Alignment, Exact, Normalizer};
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Maps {
pub a2b: Alignment,
pub b2a: Alignment,
}
#[derive(Clone, Debug)]
pub enum Input {
Tokens(Vec<String>),
Text(String),
}
#[derive(Clone, Debug)]
pub struct Case {
pub id: String,
pub group: String,
pub api: String,
pub normalizer: String,
pub a: Input,
pub b: Input,
pub expected: Maps,
pub alternatives: Vec<Maps>,
}
pub fn cases() -> Vec<Case> {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests")
.join("data")
.join("conformance-vectors.json");
let text = fs::read_to_string(path).expect("read conformance vectors");
let root = Parser::new(&text)
.parse()
.expect("parse conformance vectors");
let object = root.object();
let declared = object["case_count"].number();
let parsed: Vec<Case> = object["cases"]
.array()
.iter()
.map(Case::from_value)
.collect();
assert_eq!(declared, parsed.len(), "declared vector count");
parsed
}
pub fn run(case: &Case) -> Maps {
match (
&case.a,
&case.b,
case.api.as_str(),
case.normalizer.as_str(),
) {
(Input::Tokens(a), Input::Tokens(b), "get_alignments", "default") => {
maps(get_alignments(a, b))
}
(Input::Tokens(a), Input::Tokens(b), "get_alignments", "exact") => {
maps(get_alignments_with(a, b, &Exact))
}
(Input::Tokens(a), Input::Tokens(b), "get_alignments", "strip_atat") => {
struct StripAtAt;
impl Normalizer for StripAtAt {
fn normalize(&self, token: &str, out: &mut String) {
out.push_str(token.strip_suffix("@@").unwrap_or(token));
}
}
maps(get_alignments_with(a, b, &StripAtAt))
}
(Input::Text(a), Input::Text(b), "get_charmap", "default") => maps(get_charmap(a, b)),
_ => panic!("unsupported vector shape: {} {}", case.api, case.normalizer),
}
}
pub fn assert_case(case: &Case) {
let actual = run(case);
assert!(
actual == case.expected || case.alternatives.contains(&actual),
"case {} failed\nactual: {actual:?}\nexpected: {:?}\nalternatives: {:?}",
case.id,
case.expected,
case.alternatives
);
}
pub fn find(id: &str) -> Case {
cases()
.into_iter()
.find(|case| case.id == id)
.unwrap_or_else(|| panic!("missing vector {id}"))
}
fn maps(pair: (Alignment, Alignment)) -> Maps {
Maps {
a2b: pair.0,
b2a: pair.1,
}
}
impl Case {
fn from_value(value: &Value) -> Self {
let object = value.object();
let api = object["api"].string().to_owned();
let a = input(&object["a"], &api);
let b = input(&object["b"], &api);
Self {
id: object["id"].string().to_owned(),
group: object["group"].string().to_owned(),
api,
normalizer: object["normalizer"].string().to_owned(),
a,
b,
expected: Maps::from_value(&object["expected"]),
alternatives: object
.get("alternatives")
.map(|value| value.array().iter().map(Maps::from_value).collect())
.unwrap_or_default(),
}
}
}
impl Maps {
fn from_value(value: &Value) -> Self {
let object = value.object();
Self {
a2b: index_lists(&object["a2b"]),
b2a: index_lists(&object["b2a"]),
}
}
}
fn input(value: &Value, api: &str) -> Input {
if api == "get_charmap" {
Input::Text(value.string().to_owned())
} else {
Input::Tokens(
value
.array()
.iter()
.map(|value| value.string().to_owned())
.collect(),
)
}
}
fn index_lists(value: &Value) -> Vec<Vec<usize>> {
value
.array()
.iter()
.map(|row| row.array().iter().map(Value::number).collect())
.collect()
}
#[derive(Clone, Debug)]
enum Value {
Null,
Bool,
Number(usize),
String(String),
Array(Vec<Value>),
Object(BTreeMap<String, Value>),
}
impl Value {
fn object(&self) -> &BTreeMap<String, Value> {
match self {
Self::Object(value) => value,
_ => panic!("expected object"),
}
}
fn array(&self) -> &[Value] {
match self {
Self::Array(value) => value,
_ => panic!("expected array"),
}
}
fn string(&self) -> &str {
match self {
Self::String(value) => value,
_ => panic!("expected string"),
}
}
fn number(&self) -> usize {
match self {
Self::Number(value) => *value,
_ => panic!("expected number"),
}
}
}
struct Parser<'a> {
text: &'a str,
position: usize,
}
impl<'a> Parser<'a> {
fn new(text: &'a str) -> Self {
Self { text, position: 0 }
}
fn parse(mut self) -> Result<Value, String> {
let value = self.value()?;
self.whitespace();
if self.position == self.text.len() {
Ok(value)
} else {
Err(self.error("trailing data"))
}
}
fn value(&mut self) -> Result<Value, String> {
self.whitespace();
match self.peek() {
Some(b'n') => {
self.literal("null")?;
Ok(Value::Null)
}
Some(b't') => {
self.literal("true")?;
Ok(Value::Bool)
}
Some(b'f') => {
self.literal("false")?;
Ok(Value::Bool)
}
Some(b'0'..=b'9') => self.number(),
Some(b'"') => self.string().map(Value::String),
Some(b'[') => self.array(),
Some(b'{') => self.object(),
_ => Err(self.error("expected value")),
}
}
fn array(&mut self) -> Result<Value, String> {
self.expect(b'[')?;
let mut values = Vec::new();
self.whitespace();
if self.consume(b']') {
return Ok(Value::Array(values));
}
loop {
values.push(self.value()?);
self.whitespace();
if self.consume(b']') {
return Ok(Value::Array(values));
}
self.expect(b',')?;
}
}
fn object(&mut self) -> Result<Value, String> {
self.expect(b'{')?;
let mut values = BTreeMap::new();
self.whitespace();
if self.consume(b'}') {
return Ok(Value::Object(values));
}
loop {
self.whitespace();
let key = self.string()?;
self.whitespace();
self.expect(b':')?;
values.insert(key, self.value()?);
self.whitespace();
if self.consume(b'}') {
return Ok(Value::Object(values));
}
self.expect(b',')?;
}
}
fn string(&mut self) -> Result<String, String> {
self.expect(b'"')?;
let mut value = String::new();
loop {
let byte = self
.peek()
.ok_or_else(|| self.error("unterminated string"))?;
match byte {
b'"' => {
self.position += 1;
return Ok(value);
}
b'\\' => {
self.position += 1;
let escaped = self
.peek()
.ok_or_else(|| self.error("unterminated escape"))?;
self.position += 1;
match escaped {
b'"' => value.push('"'),
b'\\' => value.push('\\'),
b'/' => value.push('/'),
b'b' => value.push('\u{0008}'),
b'f' => value.push('\u{000C}'),
b'n' => value.push('\n'),
b'r' => value.push('\r'),
b't' => value.push('\t'),
b'u' => value.push(self.unicode_escape()?),
_ => return Err(self.error("invalid escape")),
}
}
0..=0x1F => return Err(self.error("control character in string")),
_ => {
let character = self.text[self.position..]
.chars()
.next()
.ok_or_else(|| self.error("invalid character"))?;
value.push(character);
self.position += character.len_utf8();
}
}
}
}
fn unicode_escape(&mut self) -> Result<char, String> {
let first = self.hex_quad()?;
if (0xD800..=0xDBFF).contains(&first) {
self.expect(b'\\')?;
self.expect(b'u')?;
let second = self.hex_quad()?;
if !(0xDC00..=0xDFFF).contains(&second) {
return Err(self.error("invalid low surrogate"));
}
let codepoint = 0x10000 + ((first - 0xD800) << 10) + (second - 0xDC00);
char::from_u32(codepoint).ok_or_else(|| self.error("invalid code point"))
} else {
char::from_u32(first).ok_or_else(|| self.error("invalid code point"))
}
}
fn hex_quad(&mut self) -> Result<u32, String> {
let end = self.position.saturating_add(4);
let digits = self
.text
.get(self.position..end)
.ok_or_else(|| self.error("short Unicode escape"))?;
self.position = end;
u32::from_str_radix(digits, 16).map_err(|_| self.error("invalid Unicode escape"))
}
fn number(&mut self) -> Result<Value, String> {
let start = self.position;
while matches!(self.peek(), Some(b'0'..=b'9')) {
self.position += 1;
}
self.text[start..self.position]
.parse()
.map(Value::Number)
.map_err(|_| self.error("invalid number"))
}
fn literal(&mut self, expected: &str) -> Result<(), String> {
if self.text[self.position..].starts_with(expected) {
self.position += expected.len();
Ok(())
} else {
Err(self.error("invalid literal"))
}
}
fn expect(&mut self, expected: u8) -> Result<(), String> {
if self.consume(expected) {
Ok(())
} else {
Err(self.error("unexpected byte"))
}
}
fn consume(&mut self, expected: u8) -> bool {
if self.peek() == Some(expected) {
self.position += 1;
true
} else {
false
}
}
fn peek(&self) -> Option<u8> {
self.text.as_bytes().get(self.position).copied()
}
fn whitespace(&mut self) {
while matches!(self.peek(), Some(b' ' | b'\n' | b'\r' | b'\t')) {
self.position += 1;
}
}
fn error(&self, message: &str) -> String {
format!("{message} at byte {}", self.position)
}
}