use super::*;
use crate::lexer::is_ink_ident_codepoint;
use SyntaxKind::*;
#[test]
fn cjk_identifier() {
assert_eq!(tokens("日本語"), vec![(IDENT, "日本語")]);
}
#[test]
fn latin_extended() {
assert_eq!(tokens("foo_Ā"), vec![(IDENT, "foo_Ā")]); }
#[test]
fn latin1_supplement_is_valid() {
assert_eq!(tokens("café"), vec![(IDENT, "café")]);
}
#[test]
fn cyrillic_identifier() {
assert_eq!(tokens("привет"), vec![(IDENT, "привет")]);
}
#[test]
fn arabic_identifier() {
assert_eq!(tokens("مرحبا"), vec![(IDENT, "مرحبا")]);
}
#[test]
fn hebrew_identifier() {
assert_eq!(tokens("שלום"), vec![(IDENT, "שלום")]);
}
#[test]
fn korean_identifier() {
assert_eq!(tokens("안녕"), vec![(IDENT, "안녕")]);
}
#[test]
fn hiragana_identifier() {
assert_eq!(tokens("こんにちは"), vec![(IDENT, "こんにちは")]);
}
#[test]
fn katakana_identifier() {
assert_eq!(tokens("カタカナ"), vec![(IDENT, "カタカナ")]);
}
#[test]
fn mixed_ascii_and_unicode() {
assert_eq!(tokens("myVar_日本語"), vec![(IDENT, "myVar_日本語")]);
}
#[test]
fn outside_ranges_is_error() {
let toks = tokens("☃");
assert_eq!(toks[0].0, ERROR_TOKEN);
}
#[test]
fn emoji_is_error() {
let toks = tokens("🎉");
assert_eq!(toks[0].0, ERROR_TOKEN);
}
#[test]
fn greek_with_exclusions() {
assert!(is_ink_ident_codepoint('\u{0370}')); assert!(is_ink_ident_codepoint('\u{0386}')); assert!(is_ink_ident_codepoint('\u{03FF}'));
assert!(!is_ink_ident_codepoint('\u{0374}')); assert!(!is_ink_ident_codepoint('\u{0375}')); assert!(!is_ink_ident_codepoint('\u{0378}')); assert!(!is_ink_ident_codepoint('\u{0385}')); assert!(!is_ink_ident_codepoint('\u{0387}')); assert!(!is_ink_ident_codepoint('\u{038B}')); assert!(!is_ink_ident_codepoint('\u{038D}')); assert!(!is_ink_ident_codepoint('\u{03A2}')); }
#[test]
fn cyrillic_with_exclusions() {
assert!(is_ink_ident_codepoint('\u{0400}')); assert!(is_ink_ident_codepoint('\u{0481}')); assert!(is_ink_ident_codepoint('\u{048A}')); assert!(is_ink_ident_codepoint('\u{04FF}'));
assert!(!is_ink_ident_codepoint('\u{0482}')); assert!(!is_ink_ident_codepoint('\u{0489}')); }
#[test]
fn armenian_with_exclusions() {
assert!(is_ink_ident_codepoint('\u{0531}')); assert!(is_ink_ident_codepoint('\u{0556}')); assert!(is_ink_ident_codepoint('\u{0561}')); assert!(is_ink_ident_codepoint('\u{0587}')); assert!(is_ink_ident_codepoint('\u{058F}'));
assert!(!is_ink_ident_codepoint('\u{0530}')); assert!(!is_ink_ident_codepoint('\u{0557}')); assert!(!is_ink_ident_codepoint('\u{0560}')); assert!(!is_ink_ident_codepoint('\u{0588}')); assert!(!is_ink_ident_codepoint('\u{058E}')); }
#[test]
fn boundary_codepoints() {
assert!(!is_ink_ident_codepoint('\u{007F}')); assert!(!is_ink_ident_codepoint('\u{0250}')); assert!(!is_ink_ident_codepoint('\u{036F}')); assert!(!is_ink_ident_codepoint('\u{0500}')); assert!(!is_ink_ident_codepoint('\u{0700}')); assert!(!is_ink_ident_codepoint('\u{3040}')); assert!(!is_ink_ident_codepoint('\u{3097}')); assert!(!is_ink_ident_codepoint('\u{309F}')); assert!(!is_ink_ident_codepoint('\u{30FD}')); assert!(!is_ink_ident_codepoint('\u{4DFF}')); assert!(!is_ink_ident_codepoint('\u{A000}')); assert!(!is_ink_ident_codepoint('\u{ABFF}')); assert!(!is_ink_ident_codepoint('\u{D7B0}')); }