use std :: {
fmt :: { self, Display },
sync::Arc
};
use crate :: {
error :: RuleSyntaxError,
word :: { FeatKind, FeatureCategory, NodeKind, SupraKind },
CARDINALS_TRIE, DIACRITS
};
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub(crate) enum TokenKind {
LeftSquare, RightSquare, LeftCurly, RightCurly, RightAngle, LeftAngle, LeftBracket, RightBracket, LeftColCurly, RightColCurly, GreaterThan, Equals, Underline, Arrow, Reverse, Comma, Colon, ExternBoundary, WordBoundary, SyllBoundary, Syllable, Ampersand, AtSign, Group, Number, Slash, Pipe, Cardinal, Diacritic(u8), Star, EmptySet, Ellipsis, WrappedEllipsis, Comment, Feature(FeatureCategory),
Eol, }
impl TokenKind {
pub(crate) fn as_diacritic(&self) -> Option<&u8> {
if let Self::Diacritic(v) = self {
Some(v)
} else {
None
}
}
}
impl Display for TokenKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
TokenKind::LeftSquare => write!(f, "LSquare"),
TokenKind::RightSquare => write!(f, "RSquare"),
TokenKind::LeftCurly => write!(f, "LCurly"),
TokenKind::RightCurly => write!(f, "RCurly"),
TokenKind::LeftAngle => write!(f, "LAngle"),
TokenKind::RightAngle => write!(f, "RAngle"),
TokenKind::LeftBracket => write!(f, "LBrack"),
TokenKind::RightBracket => write!(f, "RBrack"),
TokenKind::LeftColCurly => write!(f, "LCCurly"),
TokenKind::RightColCurly => write!(f, "RCCurly"),
TokenKind::GreaterThan => write!(f, "GT"),
TokenKind::Equals => write!(f, "Eq"),
TokenKind::Underline => write!(f, "UL"),
TokenKind::Arrow => write!(f, "Arrow"),
TokenKind::Reverse => write!(f, "Reverse"),
TokenKind::Comma => write!(f, "Comma"),
TokenKind::Colon => write!(f, "Colon"),
TokenKind::ExternBoundary => write!(f, "XBound"),
TokenKind::WordBoundary => write!(f, "WBound"),
TokenKind::SyllBoundary => write!(f, "SBound"),
TokenKind::Syllable => write!(f, "Syll"),
TokenKind::Ampersand => write!(f, "Amper"),
TokenKind::AtSign => write!(f, "AtSign"),
TokenKind::Group => write!(f, "Prim"),
TokenKind::Number => write!(f, "Num"),
TokenKind::Slash => write!(f, "Slash"),
TokenKind::Pipe => write!(f, "Pipe"),
TokenKind::Cardinal => write!(f, "Cardinal"),
TokenKind::Diacritic(i) => write!(f, "Diacritic({i})"),
TokenKind::Star => write!(f, "Star"),
TokenKind::EmptySet => write!(f, "Empty"),
TokenKind::Ellipsis => write!(f, "Ellipsis"),
TokenKind::WrappedEllipsis => write!(f, "WrappedEllipsis"),
TokenKind::Comment => write!(f, "Comment"),
TokenKind::Feature(x) => write!(f, "{x}"),
TokenKind::Eol => write!(f, "End of Line"),
}
}
}
#[doc(hidden)]
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct Position {
pub(crate) group: usize,
pub(crate) line: usize,
pub(crate) start: usize,
pub(crate) end: usize,
}
impl Position {
pub(crate) fn new(group: usize, line: usize, start: usize, end: usize) -> Self {
Self {group, line, start, end }
}
}
impl Display for Position {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "({},{})", self.start, self.end)
}
}
#[doc(hidden)]
#[derive(Clone, PartialEq, Eq)]
pub struct Token {
pub(crate) kind: TokenKind,
pub(crate) value: Arc<str>,
pub(crate) position: Position,
}
impl Token {
pub(crate) fn new(kind: TokenKind, value: &str, group: usize, line: usize, start: usize, end: usize) -> Self {
Self { kind, value: Arc::from(value), position: Position::new(group, line, start, end) }
}
}
impl fmt::Debug for Token {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut spaces = " ".to_string();
if self.position.end <= 9 {spaces += " "}
if self.position.start <= 9 {spaces += " "}
match self.kind {
TokenKind::Feature(x) => write!(f, "{}{}`{}{}`", self.position, spaces, self.value, x),
_ => write!(f, "{}{}{} `{}`", self.position, spaces, self.kind, self.value)
}
}
}
#[derive(Default)]
pub(crate) struct Lexer<'a> {
source: &'a [char],
group: usize,
line: usize,
pos: usize,
inside_matrix: bool,
inside_option: bool,
inside_syll: bool,
inside_set: bool,
inside_env_set: bool
}
impl<'a> Lexer<'a> {
pub(crate) fn new(source: &'a [char], group: usize, line: usize) -> Self {
Self {
source,
group,
line,
pos: 0,
inside_matrix: false,
inside_option: false,
inside_syll: false,
inside_set: false,
inside_env_set: false,
}
}
fn has_more_chars(&self) -> bool { self.pos < self.source.len() }
fn trim_whitespace(&mut self) {
while self.has_more_chars() && self.source[self.pos].is_whitespace() {
self.advance();
}
}
fn chop(&mut self, n: usize) -> String {
let token = &self.source[self.pos..self.pos+n];
self.pos += n;
token.iter().collect()
}
fn chop_while<P>(&mut self, mut predicate: P) -> String where P: FnMut(&char) -> bool {
let mut n = 0;
while self.pos+n < self.source.len() && predicate(&self.source[self.pos+n]) {
n += 1;
}
self.chop(n)
}
fn curr_char(&self) -> char {
if self.has_more_chars() {
self.source[self.pos]
} else {
'\0'
}
}
fn next_char(&self) -> char {
if self.source.len() > self.pos+1 {
self.source[self.pos+1]
} else {
'\0'
}
}
fn next_next_char(&self) -> char {
if self.source.len() > self.pos+2 {
self.source[self.pos+2]
} else {
'\0'
}
}
fn last_char_eq(&self, ch: char) -> bool {
if self.pos == 0 { return false }
self.source[self.pos-1] == ch
}
fn advance(&mut self) {
self.pos += 1;
}
fn back(&mut self) {
debug_assert!(self.pos > 0);
self.pos -= 1;
}
fn get_bracket(&mut self) -> Result<Option<Token>, RuleSyntaxError> {
let start = self.pos;
let tokenkind: TokenKind;
let value;
match self.curr_char() {
')' => { tokenkind = TokenKind::RightBracket; value = ")"; self.inside_option = false },
']' => { tokenkind = TokenKind::RightSquare; value = "]"; self.inside_matrix = false },
'⟩' => { tokenkind = TokenKind::RightAngle; value = "⟩"; self.inside_syll = false },
'}' => match self.next_char() {
':' if self.next_next_char() != '[' =>
{ tokenkind = TokenKind::RightColCurly; value = "}:"; self.inside_env_set = false; self.advance(); },
_ => { tokenkind = TokenKind::RightCurly; value = "}"; self.inside_set = false; }
},
'⟨' => {
if self.inside_syll {
return Err(RuleSyntaxError::NestedBrackets(self.group, self.line, start));
}
tokenkind = TokenKind::LeftAngle; value = "⟨"; self.inside_syll = true;
},
'{' => {
if self.inside_set && !self.inside_syll {
return Err(RuleSyntaxError::NestedBrackets(self.group, self.line, start));
}
tokenkind = TokenKind::LeftCurly; value = "{"; self.inside_set = true;
},
'(' => {
if self.inside_option {
return Err(RuleSyntaxError::NestedBrackets(self.group, self.line, start));
}
tokenkind = TokenKind::LeftBracket; value = "("; self.inside_option = true;
},
'[' => {
if self.inside_matrix {
return Err(RuleSyntaxError::NestedBrackets(self.group, self.line, start));
}
tokenkind = TokenKind::LeftSquare; value = "["; self.inside_matrix = true;
},
_ => return Ok(None)
}
self.advance();
Ok(Some(Token::new(tokenkind, value, self.group, self.line, start, self.pos)))
}
fn get_primative(&mut self) -> Option<Token> {
if self.inside_matrix || !self.curr_char().is_ascii_uppercase() { return None }
let start = self.pos;
let c = self.chop(1);
Some(Token::new(TokenKind::Group, c.as_ref(), self.group, self.line, start, self.pos))
}
fn get_numeric(&mut self) -> Option<Token> {
if !self.curr_char().is_ascii_digit() { return None }
let start = self.pos;
let buffer = self.chop_while(|x| x.is_ascii_digit());
Some(Token::new(TokenKind::Number, buffer.as_str(), self.group, self.line, start, self.pos))
}
fn get_feature(&mut self) -> Result<Option<Token>, RuleSyntaxError> {
if !self.inside_matrix || self.curr_char() != '+' && self.curr_char() != '-' && !matches!(self.curr_char(), 'α'..='ω') && !self.curr_char().is_ascii_uppercase() {
return Ok(None);
}
let start = self.pos;
let val = self.curr_char();
self.advance();
let mod_val = if val == '-' && (matches!(self.curr_char(), 'α'..='ω') || self.curr_char().is_ascii_uppercase()) {
let mut tmp = String::from('-'); tmp.push(self.curr_char());
self.advance();
tmp
} else {
String::from(val)
};
self.trim_whitespace();
let mut buffer = String::new();
while self.curr_char().is_ascii_alphabetic() || self.curr_char() == '.' {
buffer.push(self.curr_char());
self.advance();
self.trim_whitespace();
}
if buffer.len() <= 1 {
return Err(RuleSyntaxError::ExpectedAlphabetic(self.curr_char(), self.group, self.line, self.pos))
}
let tkn_kind = self.feature_match(&buffer, start, self.pos)?;
if let TokenKind::Feature(FeatureCategory::Supr(SupraKind::Tone)) = tkn_kind && (mod_val == "+" || mod_val == "-") {
return Err(RuleSyntaxError::WrongModTone(self.group, self.line, start))
}
if let TokenKind::Feature(FeatureCategory::Supr(SupraKind::LengthPair)) = tkn_kind && (mod_val == "+" || mod_val == "-") {
return Err(RuleSyntaxError::FeatCannotBeBinary(buffer, self.group, self.line, start))
}
if let TokenKind::Feature(FeatureCategory::Supr(SupraKind::StressPair)) = tkn_kind && (mod_val == "+" || mod_val == "-") {
return Err(RuleSyntaxError::FeatCannotBeBinary(buffer, self.group, self.line, start))
}
Ok(Some(Token::new(tkn_kind, &mod_val, self.group, self.line, start, self.pos)))
}
fn get_special_char(&mut self) -> Result<Option<Token>, RuleSyntaxError> {
let start = self.pos;
let tokenkind: TokenKind;
let value = match self.curr_char() {
',' => { tokenkind = TokenKind::Comma; self.chop(1) },
'#' => match self.next_char() {
'#' => { tokenkind = TokenKind::ExternBoundary; self.chop(2) },
_ => { tokenkind = TokenKind::WordBoundary; self.chop(1) },
},
'$' => { tokenkind = TokenKind::SyllBoundary; self.chop(1) },
'%' => { tokenkind = TokenKind::Syllable; self.chop(1) },
'*' => { tokenkind = TokenKind::Star; self.chop(1) },
'∅' => { tokenkind = TokenKind::EmptySet; self.chop(1) },
'&' => { tokenkind = TokenKind::Ampersand; self.chop(1) },
'@' => { tokenkind = TokenKind::AtSign; self.chop(1) },
'_' => { tokenkind = TokenKind::Underline; self.chop_while(|c| *c == '_' ) },
':' => match self.next_char() {
'{' => {
if self.inside_env_set {
return Err(RuleSyntaxError::NestedBrackets(self.group, self.line, start));
}
tokenkind = TokenKind::LeftColCurly;
self.inside_env_set = true;
self.chop(2)
},
_ => { tokenkind = TokenKind::Colon; self.chop(1) }
},
'<' => {
if self.inside_syll {
return Err(RuleSyntaxError::NestedBrackets(self.group, self.line, start));
}
tokenkind = TokenKind::LeftAngle; self.inside_syll = true;
self.chop(1)
},
'>' => {
match self.inside_syll {
true => tokenkind = TokenKind::RightAngle,
false => tokenkind = TokenKind::GreaterThan,
}
self.inside_syll = false;
self.chop(1)
},
'|' => { tokenkind = TokenKind::Pipe; self.chop(1) },
'/' => match self.next_char() {
'/' => { tokenkind = TokenKind::Pipe; self.chop(2) },
_ => { tokenkind = TokenKind::Slash; self.chop(1) }
},
'=' => match self.next_char() {
'>' => { tokenkind = TokenKind::Arrow; self.chop(2) },
_ => { tokenkind = TokenKind::Equals; self.chop(1) },
},
'-' => match self.next_char() {
'>' => { tokenkind = TokenKind::Arrow; self.chop(2) },
_ => return Err(RuleSyntaxError::ExpectedCharArrow(self.next_char(), self.group, self.line, self.pos))
},
'~' => match self.next_char() {
'>' => { tokenkind = TokenKind::Reverse; self.chop(2) },
_ => { tokenkind = TokenKind::Reverse; self.chop(1) },
},
'…' | '⋯' => { tokenkind = TokenKind::Ellipsis; self.chop(1) },
'.' => match self.next_char() {
'.' => { tokenkind = TokenKind::Ellipsis; self.chop_while(|x| *x == '.') },
_ => return Err(RuleSyntaxError::ExpectedCharDot(self.next_char(), self.group, self.line, self.pos))
},
_ => return Ok(None)
};
Ok(Some(Token::new(tokenkind, &value, self.group, self.line, start, self.pos)))
}
fn get_diacritic(&mut self) -> Option<Token> {
if self.inside_matrix { return None }
let start = self.pos;
let char = self.cur_as_ipa();
for (i, d) in DIACRITS.iter().enumerate() {
if char == d.diacrit {
self.advance();
return Some(Token::new(TokenKind::Diacritic(i as u8), &char.to_string(), self.group, self.line, start, self.pos))
}
}
None
}
fn cur_as_ipa(&mut self) -> char {
match self.curr_char() {
'g' => 'ɡ',
'?' => 'ʔ',
'!' => 'ǃ',
'ł' => 'ɬ',
'ñ' => 'ɲ',
'φ' => 'ɸ',
'^' => match self.next_char() {
'ʘ' | 'ǀ' | 'ǁ' | 'ǃ' | '!' | '‼' | 'ǂ' |
'q' | 'ɢ' | 'ɴ' | 'χ' | 'ʁ' => { self.advance(); self.cur_as_ipa() }
_ => '^'
},
'"' => match self.next_char() {
'\'' => { self.advance(); 'ʼ' },
'j' => { self.advance(); 'ʲ' },
'w' => { self.advance(); 'ʷ' },
'g' | 'ɡ' => { self.advance(); 'ˠ' },
'h' => { self.advance(); 'ʰ' },
'ɦ' | 'H' => { self.advance(); 'ʱ' },
's' => { self.advance(); 'ˢ' },
'z' => { self.advance(); 'ᶻ' },
'l' => { self.advance(); 'ˡ' },
'r' => { self.advance(); 'ʵ'},
'm' => { self.advance(); 'ᵐ' },
'n' => { self.advance(); 'ⁿ' },
'P' | 'ñ' | 'ɲ' => { self.advance(); 'ᶮ' },
'T' | 'ɳ' => { self.advance(); 'ᶯ' },
'G' | 'ŋ' => { self.advance(); 'ᵑ' },
'N' | 'ɴ' => { self.advance(); 'ᶰ' },
'v' | 'ʋ' => { self.advance(); 'ᶹ' },
'y' | 'ɥ' => { self.advance(); 'ᶣ' },
'X' | 'χ' => { self.advance(); 'ᵡ' },
'R' | 'ʁ' => { self.advance(); 'ʶ' },
'e' | 'ə' => { self.advance(); 'ᵊ' },
'?' | 'ʔ' => { self.advance(); 'ˀ' },
_ => '"'
}
'ꭤ' => 'ɑ',
'ǝ' => 'ə',
'ℇ' => 'ɛ',
'ℎ' => 'h',
'ℏ' => 'ħ',
other => other,
}
}
fn get_ipa(&mut self) -> Option<Token> {
if self.inside_matrix { return None }
let start = self.pos;
let mut buffer = self.cur_as_ipa().to_string();
if buffer == "¢" {
buffer = "t͡s".to_string()
} else if buffer == "ƛ" {
buffer = "t͡ɬ".to_string()
} else if buffer == "λ" {
buffer = "d͡ɮ".to_string()
}
if CARDINALS_TRIE.contains_prefix(buffer.as_str()) {
self.advance();
loop {
let mut tmp = buffer.clone();
tmp.push(self.cur_as_ipa());
if CARDINALS_TRIE.contains_prefix(tmp.as_str()) {
buffer.push(self.cur_as_ipa());
self.advance();
continue;
}
if self.curr_char() == '¢' {
buffer.push_str("t͡s");
self.advance();
continue;
} else if self.curr_char() == 'ƛ' {
buffer.push_str("t͡ɬ");
self.advance();
continue;
} else if self.curr_char() == 'λ' {
buffer.push_str("d͡ɮ");
self.advance();
continue;
}
if self.curr_char() == '^' {
tmp.pop(); tmp.push('\u{0361}');
if CARDINALS_TRIE.contains_prefix(tmp.as_str()) {
buffer.push('\u{0361}');
self.advance();
continue;
}
tmp.pop(); tmp.push('\u{035C}');
if CARDINALS_TRIE.contains_prefix(tmp.as_str()) {
buffer.push('\u{035C}');
self.advance();
continue;
}
}
if self.last_char_eq('^') || self.last_char_eq('"') {
self.back();
}
return Some(Token::new(TokenKind::Cardinal, &buffer, self.group, self.line, start, self.pos))
}
}
None
}
fn get_string(&mut self) -> Result<Option<Token>, RuleSyntaxError> {
if !self.curr_char().is_ascii_alphabetic() { return Ok(None) }
if !self.inside_matrix {
return Err(RuleSyntaxError::OutsideBrackets(self.group, self.line, self.pos))
}
let start = self.pos;
let buffer = self.chop_while(|x| x.is_ascii_alphabetic());
let tkn_kind: TokenKind = self.string_match(buffer, start)?;
self.trim_whitespace();
match self.curr_char() {
':' => self.advance(),
_ => return Err(RuleSyntaxError::ExpectedCharColon(self.curr_char(), self.group, self.line, self.pos))
}
self.trim_whitespace();
match self.get_numeric() {
Some(num) => Ok(Some(Token::new(tkn_kind, &num.value, self.group, self.line, start, self.pos))),
_ => Err(RuleSyntaxError::ExpectedNumber(self.curr_char(), self.group, self.line, self.pos))
}
}
fn string_match(&mut self, buffer: String, start: usize) -> Result<TokenKind, RuleSyntaxError> {
use TokenKind::*;
use FeatureCategory::*;
use SupraKind::*;
match buffer.to_lowercase().as_str() {
"tone" | "ton" | "tne" | "tn" => Ok(Feature(Supr(Tone))),
_ => Err(RuleSyntaxError::UnknownEnbyFeature(buffer.clone(), Position::new(self.group, self.line, start, start+buffer.len())))
}
}
fn feature_match(&mut self, buffer: &String, start: usize, end: usize) -> Result<TokenKind, RuleSyntaxError> {
use FeatureCategory::*;
use NodeKind::*;
use FeatKind::*;
use SupraKind::*;
let feature = match buffer.to_lowercase().as_str() {
"root" | "rut" | "rt" => Node(Root),
"consonantal" | "consonant" | "cons" | "cns" => Feat(Consonantal),
"sonorant" | "sonor" | "son" | "snrt" | "sn" => Feat(Sonorant),
"syllabic" | "syllab" | "syll" | "syl" | "sl" => Feat(Syllabic),
"manner" | "mann" | "man" | "mnnr" | "mnr" => Node(Manner),
"continuant" | "contin" | "cont" | "cnt" => Feat(Continuant),
"approximant" | "approx" | "appr" | "app" => Feat(Approximant),
"lateral" | "latrl" | "ltrl" | "lat" | "lt" => Feat(Lateral),
"nasal" | "nsl" | "nas" | "ns" | "nl" => Feat(Nasal),
"delayedrelease" | "delrel" | "d.r." | "del.rel." |
"delayed" | "dl" | "dlrl" | "dr" | "delay" |
"drelease" | "del.rel"| "drel" | "d.r" | "dr." => Feat(DelayedRelease),
"strident" | "strid" | "stri" | "stridnt" |
"strdent" | "strdnt" => Feat(Strident),
"rhotic" | "rho" | "rhot" | "rhtc" | "rh" |
"rht" | "rhc" => Feat(Rhotic),
"click" | "clik" | "clk" | "clck" => Feat(Click),
"laryngeal" | "laryng" | "laryn" | "lar" => Node(Laryngeal),
"voice" | "voi" | "vce" | "vc" => Feat(Voice),
"spreadglottis" | "spreadglot" | "spread" | "sprd" |
"spr" | "s.g." | "s.g" | "sg." | "sg" => Feat(SpreadGlottis),
"constricted" | "constglot" | "constr" | "c.g." |
"c.g" | "cg." | "cg" | "constrictedglottis" => Feat(ConstrGlottis),
"place" | "plce" | "plc" => Node(Place),
"labial" | "lbl" | "lab" => Node(Labial),
"labiodental" | "ldental" | "labiodent" | "labio" |
"labiod" | "labdent" | "lbdntl" | "ldent" | "ldl" => Feat(Labiodental),
"round" | "rund" | "rnd" | "rd" => Feat(Round),
"coronal" | "coron" | "crnl" | "cor" => Node(Coronal),
"anterior" | "anter" | "antr" | "ant" => Feat(Anterior),
"distributed" | "distrib" | "dist" | "dis" | "dst" => Feat(Distributed),
"dorsal" | "drsl" | "dors" | "dor" => Node(Dorsal),
"front" | "frnt" | "fnt" | "fro" | "frt" | "fr" => Feat(Front),
"back" | "bck" | "bk" => Feat(Back),
"high" | "hgh" | "hi" => Feat(High),
"low" | "lw" | "lo" => Feat(Low),
"tense" | "tens" | "tns" | "ten" => Feat(Tense),
"reduced" | "reduc" | "redu" | "rdcd" | "red" => Feat(Reduced),
"pharyngeal" | "pharyng" | "pharyn" | "phar" | "phr" => Node(Pharyngeal),
"advancedtongueroot" | "a.t.r." | "a.t.r" |
"a.tr" | "at.r" | "atr" => Feat(AdvancedTongueRoot),
"retractedtongueroot" | "r.t.r." | "r.t.r" |
"r.tr" | "rt.r" | "rtr" => Feat(RetractedTongueRoot),
"long" | "lng" => Supr(Long),
"overlong" | "overlng" | "ovrlng" | "vlong" |
"olong" | "vlng" | "olng" => Supr(Overlong),
"stress" | "strs" | "str" => Supr(Stress),
"secondarystress"| "sec.stress" | "secstress" |
"sec.str." | "sec.str" | "secstr" | "sec" => Supr(SecStress),
"length" | "len" => Supr(LengthPair),
"anystress" | "anystr" | "stressany" | "strany" |
"allstress" | "allstr" | "stressall" | "strall" => Supr(StressPair),
"tone" | "ton" | "tne" | "tn" => Supr(Tone),
_ => return Err(RuleSyntaxError::UnknownFeature(buffer.to_string(), Position::new(self.group, self.line, start, end)))
};
Ok(TokenKind::Feature(feature))
}
fn get_comment(&mut self) -> Result<Option<Token>, RuleSyntaxError> {
if self.curr_char() != ';' { return Ok(None) }
let start = self.pos;
self.advance();
if self.curr_char() != ';' { return Err(RuleSyntaxError::MalformedComment(self.curr_char(), self.group, self.line, self.pos))}
self.advance();
let buffer = self.chop_while(|_| true);
Ok(Some(Token::new(TokenKind::Comment, &buffer, self.group, self.line, start, self.pos)))
}
fn get_next_token(&mut self) -> Result<Token, RuleSyntaxError> {
self.trim_whitespace();
if !self.has_more_chars() { return Ok(Token::new(TokenKind::Eol, "", self.group, self.line, self.pos, self.pos+1)) }
if let Some(com_token) = self.get_comment()? { return Ok(com_token) }
if let Some(bkt_token) = self.get_bracket()? { return Ok(bkt_token) }
if let Some(pmt_token) = self.get_primative() { return Ok(pmt_token) }
if let Some(num_token) = self.get_numeric() { return Ok(num_token) }
if let Some(ftr_token) = self.get_feature()? { return Ok(ftr_token) }
if let Some(spc_token) = self.get_special_char()? { return Ok(spc_token) }
if let Some(dia_token) = self.get_diacritic() { return Ok(dia_token) }
if let Some(ipa_token) = self.get_ipa() { return Ok(ipa_token) }
if let Some(str_token) = self.get_string()? { return Ok(str_token) }
Err(RuleSyntaxError::UnknownCharacter(self.curr_char(), self.group, self.line, self.pos))
}
pub(crate) fn get_line(&mut self) -> Result<Vec<Token>, RuleSyntaxError> {
let mut token_list: Vec<Token> = Vec::new();
loop {
let next_token = self.get_next_token()?;
match next_token.kind {
TokenKind::Eol => {
token_list.push(next_token);
break
},
TokenKind::RightBracket => match (token_list.get(token_list.len().wrapping_sub(2)), token_list.last()) {
(Some(mb), Some(me)) if mb.kind == TokenKind::LeftBracket && me.kind == TokenKind::Ellipsis => {
let tk = Token {
kind: TokenKind::WrappedEllipsis,
value: format!("({})", me.value).into(),
position: Position::new(self.group, self.line, mb.position.start, next_token.position.end)
};
token_list.pop();
token_list.pop();
token_list.push(tk);
},
_ => token_list.push(next_token)
}
_ => token_list.push(next_token)
}
}
Ok(token_list)
}
}
#[cfg(test)]
mod lexer_tests {
use super::*;
#[test]
fn test_syll() {
let test_input = "%";
let expected_result = TokenKind::Syllable;
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_next_token().unwrap();
assert_eq!(result.kind, expected_result);
assert_eq!(result.value.as_ref(), test_input);
}
#[test]
fn test_americanist_aliases() {
let test_input= String::from("¢ ñ λ ł ƛ ⁿ¢ ⁿλ ⁿƛ");
let expected_result = vec![
Token::new(TokenKind::Cardinal, "t͡s" , 0, 0, 0, 1),
Token::new(TokenKind::Cardinal, "ɲ" , 0, 0, 2, 3),
Token::new(TokenKind::Cardinal, "d͡ɮ" , 0, 0, 4, 5),
Token::new(TokenKind::Cardinal, "ɬ" , 0, 0, 6, 7),
Token::new(TokenKind::Cardinal, "t͡ɬ" , 0, 0, 8, 9),
Token::new(TokenKind::Cardinal, "ⁿt͡s" , 0, 0, 10, 12),
Token::new(TokenKind::Cardinal, "ⁿd͡ɮ" , 0, 0, 13, 15),
Token::new(TokenKind::Cardinal, "ⁿt͡ɬ" , 0, 0, 16, 18),
Token::new(TokenKind::Eol, "" ,0, 0, 18, 19),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_floating_dia() {
let test_input= String::from("\"H > \"h");
let expected_res = vec![
Token::new(TokenKind::Diacritic(29), "ʱ", 0, 0, 0, 2),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 3, 4),
Token::new(TokenKind::Diacritic(28), "ʰ", 0, 0, 5, 7),
Token::new(TokenKind::Eol, "", 0, 0, 7, 8),
];
let res = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(res.len(), expected_res.len());
for i in 0..res.len() {
assert_eq!(res[i], expected_res[i]);
}
}
#[test]
fn test_ipa_sep() {
let test_input= String::from("t͡ɕ b͡β b a ʘq ʘ^q qʘ q^ʘ b\"H");
let expected_result = vec![
Token::new(TokenKind::Cardinal, "t͡ɕ", 0, 0, 0, 3),
Token::new(TokenKind::Cardinal, "b͡β", 0, 0, 4, 7),
Token::new(TokenKind::Cardinal, "b", 0, 0, 8, 9),
Token::new(TokenKind::Cardinal, "a", 0, 0, 10, 11),
Token::new(TokenKind::Cardinal, "ʘq", 0, 0, 12, 14),
Token::new(TokenKind::Cardinal, "ʘq", 0, 0, 15, 18),
Token::new(TokenKind::Cardinal, "qʘ", 0, 0, 19, 21),
Token::new(TokenKind::Cardinal, "qʘ", 0, 0, 22, 25),
Token::new(TokenKind::Cardinal, "b", 0, 0, 26, 27),
Token::new(TokenKind::Diacritic(29), "ʱ", 0, 0, 27, 29),
Token::new(TokenKind::Eol, "", 0, 0, 29, 30),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_ipa_joined() {
let test_input= String::from("t^ɕb͡βba");
let expected_result = vec![
Token::new(TokenKind::Cardinal, "t͡ɕ", 0, 0, 0, 3),
Token::new(TokenKind::Cardinal, "b͡β", 0, 0, 3, 6),
Token::new(TokenKind::Cardinal, "b", 0, 0, 6, 7),
Token::new(TokenKind::Cardinal, "a", 0, 0, 7, 8),
Token::new(TokenKind::Eol, "", 0, 0, 8, 9),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_ipa_tie() {
let test_input= String::from("t^ɕ > b^β");
let expected_result = vec![
Token::new(TokenKind::Cardinal, "t͡ɕ", 0, 0, 0, 3),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 4, 5),
Token::new(TokenKind::Cardinal, "b͡β", 0, 0, 6, 9),
Token::new(TokenKind::Eol, "", 0, 0, 9, 10),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_metathesis() {
let test_input= String::from("t^ɕ...b͡β > &");
let expected_result = vec![
Token::new(TokenKind::Cardinal, "t͡ɕ", 0, 0, 0, 3),
Token::new(TokenKind::Ellipsis, "...", 0, 0, 3, 6),
Token::new(TokenKind::Cardinal, "b͡β", 0, 0, 6, 9),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 10, 11),
Token::new(TokenKind::Ampersand, "&", 0, 0, 12, 13),
Token::new(TokenKind::Eol, "", 0, 0, 13, 14),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
let test_input= String::from("t^ɕ(...)b͡β > &");
let expected_result = vec![
Token::new(TokenKind::Cardinal, "t͡ɕ", 0, 0, 0, 3),
Token::new(TokenKind::WrappedEllipsis, "(...)", 0, 0, 3, 8),
Token::new(TokenKind::Cardinal, "b͡β", 0, 0, 8, 11),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 12, 13),
Token::new(TokenKind::Ampersand, "&", 0, 0, 14, 15),
Token::new(TokenKind::Eol, "", 0, 0, 15, 16),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_feature_matrix() {
use FeatureCategory::*;
let test_input= String::from("[+voi, -sg, αPLACE]");
let expected_result = vec![
Token::new(TokenKind::LeftSquare, "[", 0, 0, 0, 1),
Token::new(TokenKind::Feature(Feat(FeatKind::Voice)), "+", 0, 0, 1, 5),
Token::new(TokenKind::Comma, ",", 0, 0, 5, 6),
Token::new(TokenKind::Feature(Feat(FeatKind::SpreadGlottis)), "-", 0, 0, 7, 10),
Token::new(TokenKind::Comma, ",", 0, 0, 10, 11),
Token::new(TokenKind::Feature(Node(NodeKind::Place)), "α", 0, 0, 12, 18),
Token::new(TokenKind::RightSquare, "]", 0, 0, 18, 19),
Token::new(TokenKind::Eol, "", 0, 0, 19, 20),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_negative_alpha() {
use FeatureCategory::*;
let test_input= String::from("[-αPLACE]");
let expected_result = vec![
Token::new(TokenKind::LeftSquare, "[", 0, 0, 0, 1),
Token::new(TokenKind::Feature(Node(NodeKind::Place)), "-α", 0, 0, 1, 8),
Token::new(TokenKind::RightSquare, "]", 0, 0, 8, 9),
Token::new(TokenKind::Eol, "", 0, 0, 9, 10),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_references() {
let test_input= String::from("C=1 V=2 > 2 1 / _C // ___");
let expected_result = vec![
Token::new(TokenKind::Group, "C", 0, 0, 0, 1),
Token::new(TokenKind::Equals, "=", 0, 0, 1, 2),
Token::new(TokenKind::Number, "1", 0, 0, 2, 3),
Token::new(TokenKind::Group, "V", 0, 0, 4, 5),
Token::new(TokenKind::Equals, "=", 0, 0, 5, 6),
Token::new(TokenKind::Number, "2", 0, 0, 6, 7),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 8, 9),
Token::new(TokenKind::Number, "2", 0, 0, 10, 11),
Token::new(TokenKind::Number, "1", 0, 0, 12, 13),
Token::new(TokenKind::Slash, "/", 0, 0, 14, 15),
Token::new(TokenKind::Underline, "_", 0, 0, 16, 17),
Token::new(TokenKind::Group, "C", 0, 0, 17, 18),
Token::new(TokenKind::Pipe, "//", 0, 0, 19, 21),
Token::new(TokenKind::Underline, "___", 0, 0, 22, 25),
Token::new(TokenKind::Eol, "", 0, 0, 25, 26),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_brackets() {
let test_input= String::from("{C,V} > [] / :{ j_{} }:");
let expected_result = vec![
Token::new(TokenKind::LeftCurly, "{", 0, 0, 0, 1),
Token::new(TokenKind::Group, "C", 0, 0, 1, 2),
Token::new(TokenKind::Comma, ",", 0, 0, 2, 3),
Token::new(TokenKind::Group, "V", 0, 0, 3, 4),
Token::new(TokenKind::RightCurly, "}", 0, 0, 4, 5),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 6, 7),
Token::new(TokenKind::LeftSquare, "[", 0, 0, 8, 9),
Token::new(TokenKind::RightSquare, "]", 0, 0, 9, 10),
Token::new(TokenKind::Slash, "/", 0, 0, 11, 12),
Token::new(TokenKind::LeftColCurly, ":{", 0, 0, 13, 15),
Token::new(TokenKind::Cardinal, "j", 0, 0, 16, 17),
Token::new(TokenKind::Underline, "_", 0, 0, 17, 18),
Token::new(TokenKind::LeftCurly, "{", 0, 0, 18, 19),
Token::new(TokenKind::RightCurly, "}", 0, 0, 19, 20),
Token::new(TokenKind::RightColCurly, "}:", 0, 0, 21, 23),
Token::new(TokenKind::Eol, "", 0, 0, 23, 24),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_modified_set() {
let test_input= String::from("{C,V}:[] > * / :{ j_{}:[] }:");
let expected_result = vec![
Token::new(TokenKind::LeftCurly, "{", 0, 0, 0, 1),
Token::new(TokenKind::Group, "C", 0, 0, 1, 2),
Token::new(TokenKind::Comma, ",", 0, 0, 2, 3),
Token::new(TokenKind::Group, "V", 0, 0, 3, 4),
Token::new(TokenKind::RightCurly, "}", 0, 0, 4, 5),
Token::new(TokenKind::Colon, ":", 0, 0, 5, 6),
Token::new(TokenKind::LeftSquare, "[", 0, 0, 6, 7),
Token::new(TokenKind::RightSquare, "]", 0, 0, 7, 8),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 9, 10),
Token::new(TokenKind::Star, "*", 0, 0, 11, 12),
Token::new(TokenKind::Slash, "/", 0, 0, 13, 14),
Token::new(TokenKind::LeftColCurly, ":{", 0, 0, 15, 17),
Token::new(TokenKind::Cardinal, "j", 0, 0, 18, 19),
Token::new(TokenKind::Underline, "_", 0, 0, 19, 20),
Token::new(TokenKind::LeftCurly, "{", 0, 0, 20, 21),
Token::new(TokenKind::RightCurly, "}", 0, 0, 21, 22),
Token::new(TokenKind::Colon, ":", 0, 0, 22, 23),
Token::new(TokenKind::LeftSquare, "[", 0, 0, 23, 24),
Token::new(TokenKind::RightSquare, "]", 0, 0, 24, 25),
Token::new(TokenKind::RightColCurly, "}:", 0, 0, 26, 28),
Token::new(TokenKind::Eol, "", 0, 0, 28, 29),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_external_boundary() {
let test_input= String::from("## > *");
let expected_result = vec![
Token::new(TokenKind::ExternBoundary, "##", 0, 0, 0, 2),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 3, 4),
Token::new(TokenKind::Star, "*", 0, 0, 5, 6),
Token::new(TokenKind::Eol, "", 0, 0, 6, 7),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
let test_input= String::from("O > F / V ## _ V");
let expected_result = vec![
Token::new(TokenKind::Group, "O", 0, 0, 0, 1),
Token::new(TokenKind::GreaterThan, ">", 0, 0, 2, 3),
Token::new(TokenKind::Group, "F", 0, 0, 4, 5),
Token::new(TokenKind::Slash, "/", 0, 0, 6, 7),
Token::new(TokenKind::Group, "V", 0, 0, 8, 9),
Token::new(TokenKind::ExternBoundary, "##", 0, 0, 10, 12),
Token::new(TokenKind::Underline, "_", 0, 0, 13, 14),
Token::new(TokenKind::Group, "V", 0, 0, 15, 16),
Token::new(TokenKind::Eol, "", 0, 0, 16, 17),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_underline_in_structs() {
let test_input= String::from("<a_e>");
let expected_result = vec![
Token::new(TokenKind::LeftAngle, "<", 0, 0, 0, 1),
Token::new(TokenKind::Cardinal, "a", 0, 0, 1, 2),
Token::new(TokenKind::Underline, "_", 0, 0, 2, 3),
Token::new(TokenKind::Cardinal, "e", 0, 0, 3, 4),
Token::new(TokenKind::RightAngle, ">", 0, 0, 4, 5),
Token::new(TokenKind::Eol, "", 0, 0, 5, 6),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
#[test]
fn test_sets_in_structs() {
let test_input= String::from("<{a, e}>");
let expected_result = vec![
Token::new(TokenKind::LeftAngle, "<", 0, 0, 0, 1),
Token::new(TokenKind::LeftCurly, "{", 0, 0, 1, 2),
Token::new(TokenKind::Cardinal, "a", 0, 0, 2, 3),
Token::new(TokenKind::Comma, ",", 0, 0, 3, 4),
Token::new(TokenKind::Cardinal, "e", 0, 0, 5, 6),
Token::new(TokenKind::RightCurly, "}", 0, 0, 6, 7),
Token::new(TokenKind::RightAngle, ">", 0, 0, 7, 8),
Token::new(TokenKind::Eol, "", 0, 0, 8, 9),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
let test_input= String::from("{<{a, e}>}");
let expected_result = vec![
Token::new(TokenKind::LeftCurly, "{", 0, 0, 0, 1),
Token::new(TokenKind::LeftAngle, "<", 0, 0, 1, 2),
Token::new(TokenKind::LeftCurly, "{", 0, 0, 2, 3),
Token::new(TokenKind::Cardinal, "a", 0, 0, 3, 4),
Token::new(TokenKind::Comma, ",", 0, 0, 4, 5),
Token::new(TokenKind::Cardinal, "e", 0, 0, 6, 7),
Token::new(TokenKind::RightCurly, "}", 0, 0, 7, 8),
Token::new(TokenKind::RightAngle, ">", 0, 0, 8, 9),
Token::new(TokenKind::RightCurly, "}", 0, 0, 9, 10),
Token::new(TokenKind::Eol, "", 0, 0, 10, 11),
];
let result = Lexer::new(&test_input.chars().collect::<Vec<_>>(), 0, 0).get_line().unwrap();
assert_eq!(result.len(), expected_result.len());
for i in 0..result.len() {
assert_eq!(result[i], expected_result[i]);
}
}
}