1use quick_xml::{
7 Reader,
8 events::{BytesStart, Event},
9};
10use std::{
11 borrow::Cow,
12 collections::{BTreeSet, HashMap},
13 fmt::Display,
14 num::ParseIntError,
15 path::{Path, PathBuf},
16 str::FromStr,
17};
18
19use icu_properties::props::GeneralCategory;
20
21use smol_str::SmolStr;
22
23use crate::glyphdata_bundled::{self as bundled, find_pos_by_prod_name};
24
25#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
29#[repr(u8)]
30pub enum Category {
31 Mark,
32 Space,
33 Separator,
34 Letter,
35 Number,
36 Symbol,
37 Punctuation,
38 Other,
39}
40
41#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
43#[repr(u8)]
44pub enum Subcategory {
45 Spacing,
46 Radical,
47 Math,
48 Superscript,
49 Geometry,
50 Dash,
51 DecimalDigit,
52 Currency,
53 Fraction,
54 Halfform,
55 Small,
56 Number,
57 Quote,
58 Space,
59 Letter,
60 Jamo,
61 Format,
62 Parenthesis,
63 Matra,
64 Arrow,
65 Nonspacing,
66 Compatibility,
67 Syllable,
68 Ligature,
69 Modifier,
70 SpacingCombining,
71 Emoji,
72 Enclosing,
73 Composition,
74 Lowercase,
75 Uppercase,
76 Smallcaps,
77 Conjunct,
78 Other,
79}
80
81#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
83#[repr(u8)]
84pub enum Script {
85 Adlam,
86 Alchemical,
87 Arabic,
88 Armenian,
89 Avestan,
90 Balinese,
91 Bamum,
92 Batak,
93 Bengali,
94 BlackLetter,
95 Bopomofo,
96 Brahmi,
97 Braille,
98 Buginese,
99 Canadian,
100 Chakma,
101 Cham,
102 Cherokee,
103 Chorasmian,
104 Coptic,
105 Cyrillic,
106 Dentistry,
107 Deseret,
108 Devanagari,
109 Divesakuru,
110 Elbasan,
111 Elymaic,
112 Ethiopic,
113 Georgian,
114 Glagolitic,
115 Gothic,
116 Greek,
117 Gujarati,
118 Gurmukhi,
119 Han,
120 Hangul,
121 Hebrew,
122 Javanese,
123 Kana,
124 Kannada,
125 Kawi,
126 Kayahli,
127 Khmer,
128 Khojki,
129 Lao,
130 Latin,
131 Lepcha,
132 Lue,
133 Mahjong,
134 Malayalam,
135 Mandaic,
136 Math,
137 Mongolian,
138 Musical,
139 Myanmar,
140 Nko,
141 NyiakengPuachueHmong,
142 Ogham,
143 OldGeorgian,
144 Oriya,
145 Osage,
146 Osmanya,
147 PahawhHmong,
148 PhaistosDisc,
149 Rovas,
150 Runic,
151 Samaritan,
152 Shavian,
153 Sinhala,
154 Syriac,
155 Tagalog,
156 Tamil,
157 Telugu,
158 Thaana,
159 Thai,
160 Tham,
161 Tibet,
162 Tifinagh,
163 TuluTigalari,
164 Vai,
165 Yezidi,
166 Yi,
167}
168
169#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
176pub enum ProductionName {
177 Bmp(u32),
179 NonBmp(u32),
181 Custom(SmolStr),
183}
184
185impl From<&str> for ProductionName {
186 fn from(v: &str) -> ProductionName {
187 fn try_parse(
188 v: &str,
189 lbound: u32,
190 ubound: u32,
191 f: impl Fn(u32) -> ProductionName,
192 ) -> Option<ProductionName> {
193 if let Ok(v) = u32::from_str_radix(v, 16)
194 && v >= lbound
195 && v <= ubound
196 {
197 return Some(f(v));
198 }
199 None
200 }
201
202 match v {
203 _ if v.starts_with("uni") => try_parse(&v[3..], 0, 0xFFFF, ProductionName::Bmp),
204 _ if v.starts_with("u") => {
205 try_parse(&v[1..], 0xFFFF + 1, 0x10FFFF, ProductionName::NonBmp)
206 }
207 _ => None,
208 }
209 .unwrap_or_else(|| ProductionName::Custom(v.into()))
210 }
211}
212
213impl From<u32> for ProductionName {
214 fn from(v: u32) -> ProductionName {
215 if v <= 0xFFFF {
216 ProductionName::Bmp(v)
217 } else {
218 ProductionName::NonBmp(v)
219 }
220 }
221}
222
223impl Display for ProductionName {
224 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
225 match self {
226 ProductionName::Bmp(cp) => write!(f, "uni{cp:04X}"),
227 ProductionName::NonBmp(cp) => write!(f, "u{cp:X}"),
228 ProductionName::Custom(s) => write!(f, "{s}"),
229 }
230 }
231}
232
233impl From<ProductionName> for SmolStr {
234 fn from(v: ProductionName) -> SmolStr {
235 match v {
236 ProductionName::Bmp(cp) => smol_str::format_smolstr!("uni{cp:04X}"),
237 ProductionName::NonBmp(cp) => smol_str::format_smolstr!("u{cp:X}"),
238 ProductionName::Custom(s) => s,
239 }
240 }
241}
242
243#[derive(Default)]
249pub struct GlyphData {
250 overrides: Option<HashMap<SmolStr, QueryResult>>,
252 overrrides_by_codepoint: Option<HashMap<u32, SmolStr>>,
253}
254
255impl GlyphData {
256 pub fn new(overrides: Option<HashMap<SmolStr, QueryResult>>) -> Self {
258 let overrrides_by_codepoint = overrides.as_ref().map(|overrides| {
259 overrides
260 .iter()
261 .filter_map(|(k, v)| v.codepoint.map(|cp| (cp, k.clone())))
262 .collect()
263 });
264 Self {
265 overrides,
266 overrrides_by_codepoint,
267 }
268 }
269
270 pub fn with_override_file(override_file: &Path) -> Result<Self, GlyphDataError> {
272 let bytes = std::fs::read(override_file).map_err(|err| GlyphDataError::UserFile {
273 path: override_file.to_owned(),
274 reason: err.kind(),
275 })?;
276 let overrides = parse_entries(&bytes)?;
277 Ok(GlyphData::new(Some(overrides)))
278 }
279}
280
281#[derive(Debug, Clone, PartialEq)]
285pub struct QueryResult {
286 pub category: Category,
287 pub subcategory: Option<Subcategory>,
288 pub codepoint: Option<u32>,
289 pub script: Option<Script>,
290 pub production_name: Option<ProductionName>,
291}
292
293#[derive(Clone, Debug, thiserror::Error)]
294#[non_exhaustive]
295pub enum GlyphDataError {
296 #[error("Couldn't read user file at '{path}': '{reason}'")]
297 UserFile {
298 path: PathBuf,
299 reason: std::io::ErrorKind,
300 },
301 #[error("Error parsing XML: '{0}'")]
302 ReaderError(#[from] quick_xml::Error),
303 #[error("Error parsing XML attribute: '{0}'")]
304 XmlAttributeError(#[from] quick_xml::events::attributes::AttrError),
305 #[error("Unknown category '{0}'")]
306 InvalidCategory(SmolStr),
307 #[error("Unknown subcategory '{0}'")]
308 InvalidSubcategory(SmolStr),
309 #[error("Unknown script '{0}'")]
310 InvalidScript(SmolStr),
311 #[error("the XML input did not start with a <glyphdata> tag")]
312 WrongFirstElement,
313 #[error("Missing required attribute '{missing}' in '{attributes}'")]
314 MissingRequiredAttribute {
315 attributes: String,
316 missing: &'static str,
317 },
318 #[error("Invalid unicode value '{raw}': '{inner}'")]
319 InvalidUnicode { raw: String, inner: ParseIntError },
320 #[error("Unexpected attribute '{0}'")]
321 UnknownAttribute(String),
322}
323
324impl GlyphDataError {
325 fn missing_attr(name: &'static str, raw_attrs: &[u8]) -> Self {
327 let attributes = String::from_utf8_lossy(raw_attrs).into_owned();
328 Self::MissingRequiredAttribute {
329 attributes,
330 missing: name,
331 }
332 }
333}
334
335pub(crate) fn parse_entries(xml: &[u8]) -> Result<HashMap<SmolStr, QueryResult>, GlyphDataError> {
337 fn check_and_advance_past_preamble(reader: &mut Reader<&[u8]>) -> Result<(), GlyphDataError> {
338 loop {
339 let event = reader.read_event()?;
340 match event {
341 Event::Comment(_) => (),
342 Event::Decl(_) => (),
343 Event::DocType(_) => (),
344 Event::Start(start) if start.name().as_ref() == b"glyphData" => return Ok(()),
345 _other => {
346 return Err(GlyphDataError::WrongFirstElement);
347 }
348 }
349 }
350 }
351
352 let mut reader = Reader::from_reader(xml);
353 reader.config_mut().trim_text(true);
354
355 check_and_advance_past_preamble(&mut reader)?;
356
357 let mut by_name = HashMap::new();
358 let mut alt_names = Vec::new();
359 for result in
360 iter_rows(&mut reader).map(|row| row.map_err(Into::into).and_then(parse_glyph_xml))
361 {
362 let info = result?;
363 by_name.insert(
364 info.name.clone(),
365 QueryResult {
366 category: info.category,
367 subcategory: info.subcategory,
368 codepoint: info.codepoint,
369 script: info.script,
370 production_name: info.production_name.clone(),
371 },
372 );
373 for alt in info.alt_names {
374 alt_names.push((
375 alt,
376 QueryResult {
377 category: info.category,
378 subcategory: info.subcategory,
379 codepoint: None,
380 script: info.script,
381 production_name: info.production_name.clone(),
382 },
383 ));
384 }
385 }
386
387 for (name, value) in alt_names {
389 by_name.entry(name).or_insert(value);
390 }
391
392 Ok(by_name)
393}
394
395fn iter_rows<'a, 'b: 'a>(
396 reader: &'b mut Reader<&'a [u8]>,
397) -> impl Iterator<Item = Result<BytesStart<'a>, quick_xml::Error>> + 'a {
398 std::iter::from_fn(|| match reader.read_event() {
399 Err(e) => Some(Err(e)),
400 Ok(Event::Empty(start)) => Some(Ok(start)),
401 _ => None,
402 })
403}
404
405struct GlyphInfoFromXml {
406 name: SmolStr,
407 alt_names: Vec<SmolStr>,
408 category: Category,
409 subcategory: Option<Subcategory>,
410 codepoint: Option<u32>,
411 script: Option<Script>,
412 production_name: Option<ProductionName>,
413}
414
415fn parse_glyph_xml(item: BytesStart) -> Result<GlyphInfoFromXml, GlyphDataError> {
416 let mut name = None;
417 let mut category = None;
418 let mut subcategory = None;
419 let mut unicode = None;
420 let mut alt_names = None;
421 let mut script = None;
422 let mut production_name = None;
423
424 for attr in item.attributes() {
425 let attr = attr?;
426 let value = attr.normalized_value(quick_xml::XmlVersion::Implicit1_0)?;
429 match attr.key.as_ref() {
430 b"name" => name = Some(value),
431 b"category" => category = Some(value),
432 b"subCategory" => subcategory = Some(value),
433 b"unicode" => unicode = Some(value),
434 b"altNames" => alt_names = Some(value),
435 b"script" => script = Some(value),
436 b"production" => production_name = Some(value.as_ref().into()),
437 b"unicodeLegacy" | b"case" | b"direction" | b"description" => (),
438 other => {
439 return Err(GlyphDataError::UnknownAttribute(
440 String::from_utf8_lossy(other).into_owned(),
441 ));
442 }
443 }
444 }
445
446 let name = name
448 .map(SmolStr::new)
449 .ok_or_else(|| GlyphDataError::missing_attr("name", item.attributes_raw()))?;
450 let category = category
451 .ok_or_else(|| GlyphDataError::missing_attr("category", item.attributes_raw()))
452 .and_then(|cat| {
453 Category::from_str(cat.as_ref()).map_err(GlyphDataError::InvalidCategory)
454 })?;
455 let subcategory = subcategory
456 .map(|cat| Subcategory::from_str(cat.as_ref()).map_err(GlyphDataError::InvalidSubcategory))
457 .transpose()?;
458 let script = script
459 .map(|cat| Script::from_str(cat.as_ref()).map_err(GlyphDataError::InvalidScript))
460 .transpose()?;
461 let codepoint = unicode
462 .map(|s| {
463 u32::from_str_radix(&s, 16).map_err(|inner| GlyphDataError::InvalidUnicode {
464 raw: s.into_owned(),
465 inner,
466 })
467 })
468 .transpose()?;
469 let alt_names = alt_names
470 .map(|names| {
471 names
472 .as_ref()
473 .split(',')
474 .map(|name| SmolStr::from(name.trim()))
475 .collect()
476 })
477 .unwrap_or_default();
478
479 Ok(GlyphInfoFromXml {
480 name,
481 alt_names,
482 category,
483 subcategory,
484 codepoint,
485 script,
486 production_name,
487 })
488}
489
490impl GlyphData {
491 pub fn query(&self, name: &str, codepoints: Option<&BTreeSet<u32>>) -> Option<QueryResult> {
501 self.query_no_synthesis(name, codepoints)
502 .or_else(|| self.construct_result(name))
504 }
505
506 fn query_no_synthesis(
510 &self,
511 name: &str,
512 codepoints: Option<&BTreeSet<u32>>,
513 ) -> Option<QueryResult> {
514 if let (Some(overrides), Some(overrides_by_codepoint)) = (
516 self.overrides.as_ref(),
517 self.overrrides_by_codepoint.as_ref(),
518 ) {
519 let override_result = overrides.get(name).or_else(|| {
520 codepoints
521 .into_iter()
522 .flat_map(|cps| cps.iter())
523 .find_map(|cp: &u32| {
524 overrides_by_codepoint
525 .get(cp)
526 .and_then(|n| overrides.get(n))
527 })
528 });
529 if let Some(override_result) = override_result {
530 return Some(QueryResult {
531 category: override_result.category,
532 subcategory: override_result.subcategory,
533 codepoint: override_result.codepoint,
534 script: override_result.script,
535 production_name: override_result.production_name.clone(),
536 });
537 }
538 }
539
540 bundled::find_pos_by_name(name)
542 .or_else(|| find_pos_by_prod_name(name.into()))
543 .or_else(|| {
544 codepoints
545 .into_iter()
546 .flat_map(|cps| cps.iter())
547 .find_map(|cp| bundled::find_pos_by_codepoint(*cp))
548 })
549 .map(|i| {
550 bundled::get(i).unwrap_or_else(|| panic!("We found invalid index {i} somehow"))
551 })
552 }
553
554 fn contains_name(&self, name: &str) -> bool {
555 if let Some(overrides) = self.overrides.as_ref() {
556 let name: SmolStr = name.into();
557 if overrides.contains_key(&name) {
558 return true;
559 }
560 }
561 bundled::find_pos_by_name(name).is_some()
562 }
563
564 fn construct_result(&self, name: &str) -> Option<QueryResult> {
565 let category_subcategory = self.construct_category(name);
566 let production_name = self.construct_production_name(name);
567 if category_subcategory.is_none() && production_name.is_none() {
568 return None;
569 }
570 let (category, subcategory) = category_subcategory.unwrap_or((Category::Other, None));
572 Some(QueryResult {
573 category,
574 subcategory,
575 codepoint: None,
576 script: None,
577 production_name,
578 })
579 }
580
581 fn construct_category(&self, name: &str) -> Option<(Category, Option<Subcategory>)> {
583 if name.starts_with('_') {
585 return None;
586 }
587 let (base_name, _) = self.split_glyph_suffix(name);
588 if let Some(result) = self.query_no_synthesis(base_name, None) {
589 return Some((result.category, result.subcategory));
590 }
591
592 if let Some(base_names) = self.split_ligature_glyph_name(base_name) {
593 let base_names_attributes: Vec<_> = base_names
594 .iter()
595 .map(|name| self.query_no_synthesis(name, None))
596 .collect();
597 if let Some(first_attr) = base_names_attributes
598 .first()
599 .expect("if we have base_names it is non-empty")
600 {
601 if first_attr.category == Category::Mark {
603 return Some((Category::Mark, first_attr.subcategory));
604 } else if first_attr.category == Category::Letter {
605 if base_names_attributes
607 .iter()
608 .skip(1)
609 .map(|result| result.as_ref().map(|r| r.category))
610 .all(|cat| matches!(cat, None | Some(Category::Mark | Category::Separator)))
611 {
612 return Some((first_attr.category, first_attr.subcategory));
613 } else {
614 return Some((Category::Letter, Some(Subcategory::Ligature)));
615 }
616 }
617 }
618 };
619
620 Self::construct_category_via_agl(base_name)
622 }
623
624 fn construct_production_name(&self, name: &str) -> Option<ProductionName> {
626 fn append_suffix(base_name: &mut String, suffix: Option<&str>) {
627 if let Some(suffix) = suffix {
628 base_name.push('.');
629 base_name.push_str(suffix);
630 }
631 }
632
633 fn is_u_name(name: &str) -> bool {
634 name.starts_with("u") && name[1..].bytes().all(|b| b.is_ascii_hexdigit())
635 }
636
637 let (base_name, suffix) = self.split_glyph_suffix(name);
638
639 let prod_name_with_suffix = suffix.and_then(|_| {
641 self.query_no_synthesis(base_name, None)
642 .and_then(|result| result.production_name)
643 .map(|base_prod_name| {
644 let mut prod_name = base_prod_name.to_string();
645 append_suffix(&mut prod_name, suffix);
646 prod_name.as_str().into()
647 })
648 });
649 if prod_name_with_suffix.is_some() {
650 return prod_name_with_suffix;
651 }
652
653 let base_names = self
654 .split_ligature_glyph_name(base_name)
655 .unwrap_or_else(|| vec![base_name.into()]);
656 let prod_names: Vec<SmolStr> = base_names
660 .into_iter()
661 .map(|name| {
662 self.query_no_synthesis(&name, None).and_then(|result| {
663 result.production_name.map(Into::into).or_else(|| {
664 fontdrasil::agl::char_for_agl_name(name.as_ref()).map(|_| name)
666 })
667 })
668 })
669 .collect::<Option<_>>()?;
670
671 let any_characters_outside_bmp = prod_names
675 .iter()
676 .any(|name| name.len() > 5 && is_u_name(name.as_ref()));
677 let any_uni_names = prod_names.iter().any(|name| name.starts_with("uni"));
678
679 if !any_characters_outside_bmp && any_uni_names {
680 let mut uni_names: Vec<Cow<str>> = Vec::new();
681 for part in &prod_names {
682 if let Some(stripped) = part.strip_prefix("uni") {
683 uni_names.push(Cow::Borrowed(stripped));
684 } else if part.len() == 5 && is_u_name(part.as_ref()) {
685 uni_names.push(Cow::Borrowed(&part.as_str()[1..]));
686 } else if let Some(ch) = fontdrasil::agl::char_for_agl_name(part.as_ref()) {
687 uni_names.push(Cow::Owned(format!("{:04X}", ch as u32)));
688 } else {
689 panic!("Unexpected part while constructing production name: {part}");
690 }
691 }
692 let mut result = String::from("uni");
693 for segment in uni_names {
694 result.push_str(segment.as_ref());
695 }
696 append_suffix(&mut result, suffix);
697 return Some(result.as_str().into());
698 }
699
700 let mut result = prod_names.join("_");
701 append_suffix(&mut result, suffix);
702 Some(result.as_str().into())
703 }
704
705 fn construct_category_via_agl(base_name: &str) -> Option<(Category, Option<Subcategory>)> {
708 if let Some(first_char) = fontdrasil::agl::glyph_name_to_unicode(base_name)
709 .chars()
710 .next()
711 {
712 let (category, subcategory) = category_from_icu(first_char);
713
714 if base_name.contains('_') && category != Category::Mark {
717 return Some((category, Some(Subcategory::Ligature)));
718 } else {
719 return Some((category, subcategory));
720 }
721 }
722 None
723 }
724
725 fn split_glyph_suffix<'n>(&self, name: &'n str) -> (&'n str, Option<&'n str>) {
726 let multi_suffix = name.bytes().filter(|b| *b == b'.').count() > 1;
727 if multi_suffix {
728 for idx in name
733 .bytes()
734 .enumerate()
735 .filter_map(|(i, b)| (b == b'.').then_some(i))
736 .skip(1)
737 {
738 let (base, suffix) = name.split_at(idx);
739 if self.contains_name(base) {
740 return (base, Some(&suffix[1..]));
742 }
743 }
744 }
745 name.split_once('.')
747 .map_or_else(|| (name, None), |(base, suffix)| (base, Some(suffix)))
748 }
749
750 fn split_ligature_glyph_name(&self, name: &str) -> Option<Vec<SmolStr>> {
757 let script_suffix = name.rsplit_once('_')?.1.rsplit_once('-').map(|(_, x)| x);
759
760 let mut parts: Vec<_> = name
761 .trim_end_matches(script_suffix.unwrap_or_default())
762 .trim_end_matches('-')
764 .split('_')
765 .map(SmolStr::new)
766 .collect();
767
768 let script = match script_suffix {
769 None => return Some(parts),
771 Some(script) => script,
772 };
773
774 for part in parts.iter_mut() {
777 if part.contains('-') {
779 continue;
780 }
781
782 let new_part = smol_str::format_smolstr!("{part}-{script}");
783 if self.contains_name(part.as_ref()) && !self.contains_name(&new_part) {
785 continue;
786 }
787 *part = new_part;
788 }
789 Some(parts)
790 }
791}
792
793fn category_from_icu(c: char) -> (Category, Option<Subcategory>) {
795 match icu_properties::CodePointMapData::<GeneralCategory>::new().get(c) {
796 GeneralCategory::Unassigned | GeneralCategory::OtherSymbol => (Category::Symbol, None),
797 GeneralCategory::UppercaseLetter
798 | GeneralCategory::LowercaseLetter
799 | GeneralCategory::TitlecaseLetter
800 | GeneralCategory::OtherLetter => (Category::Letter, None),
801 GeneralCategory::ModifierLetter => (Category::Letter, Some(Subcategory::Modifier)),
802 GeneralCategory::NonspacingMark => (Category::Mark, Some(Subcategory::Nonspacing)),
803 GeneralCategory::SpacingMark => (Category::Mark, Some(Subcategory::SpacingCombining)),
804 GeneralCategory::EnclosingMark => (Category::Mark, Some(Subcategory::Enclosing)),
805 GeneralCategory::DecimalNumber | GeneralCategory::OtherNumber => {
806 (Category::Number, Some(Subcategory::DecimalDigit))
807 }
808 GeneralCategory::LetterNumber => (Category::Number, None),
809 GeneralCategory::SpaceSeparator => (Category::Separator, Some(Subcategory::Space)),
810 GeneralCategory::LineSeparator
811 | GeneralCategory::ParagraphSeparator
812 | GeneralCategory::Control => (Category::Separator, None),
813 GeneralCategory::Format => (Category::Separator, Some(Subcategory::Format)),
814 GeneralCategory::PrivateUse => (Category::Letter, Some(Subcategory::Compatibility)),
815 GeneralCategory::DashPunctuation => (Category::Punctuation, Some(Subcategory::Dash)),
816 GeneralCategory::OpenPunctuation | GeneralCategory::ClosePunctuation => {
817 (Category::Punctuation, Some(Subcategory::Parenthesis))
818 }
819 GeneralCategory::ConnectorPunctuation | GeneralCategory::OtherPunctuation => {
820 (Category::Punctuation, None)
821 }
822 GeneralCategory::InitialPunctuation | GeneralCategory::FinalPunctuation => {
823 (Category::Punctuation, Some(Subcategory::Quote))
824 }
825 GeneralCategory::MathSymbol => (Category::Symbol, Some(Subcategory::Math)),
826 GeneralCategory::CurrencySymbol => (Category::Symbol, Some(Subcategory::Currency)),
827 GeneralCategory::ModifierSymbol => (Category::Mark, Some(Subcategory::Spacing)),
828 GeneralCategory::Surrogate => unreachable!("char cannot represent surrogate code points"),
829 }
830}
831
832impl FromStr for Category {
833 type Err = SmolStr;
834
835 fn from_str(s: &str) -> Result<Self, Self::Err> {
836 match s {
837 "Mark" => Ok(Self::Mark),
838 "Space" => Ok(Self::Space),
839 "Separator" => Ok(Self::Separator),
840 "Letter" => Ok(Self::Letter),
841 "Number" => Ok(Self::Number),
842 "Symbol" => Ok(Self::Symbol),
843 "Punctuation" => Ok(Self::Punctuation),
844 "Other" => Ok(Self::Other),
845 _ => Err(s.into()),
846 }
847 }
848}
849
850impl FromStr for Subcategory {
851 type Err = SmolStr;
852
853 fn from_str(s: &str) -> Result<Self, Self::Err> {
854 match s {
855 "Spacing" => Ok(Self::Spacing),
856 "Radical" => Ok(Self::Radical),
857 "Math" => Ok(Self::Math),
858 "Superscript" => Ok(Self::Superscript),
859 "Geometry" => Ok(Self::Geometry),
860 "Dash" => Ok(Self::Dash),
861 "Decimal Digit" => Ok(Self::DecimalDigit),
862 "Currency" => Ok(Self::Currency),
863 "Fraction" => Ok(Self::Fraction),
864 "Halfform" => Ok(Self::Halfform),
865 "Small" => Ok(Self::Small),
866 "Number" => Ok(Self::Number),
867 "Quote" => Ok(Self::Quote),
868 "Space" => Ok(Self::Space),
869 "Letter" => Ok(Self::Letter),
870 "Jamo" => Ok(Self::Jamo),
871 "Format" => Ok(Self::Format),
872 "Parenthesis" => Ok(Self::Parenthesis),
873 "Matra" => Ok(Self::Matra),
874 "Arrow" => Ok(Self::Arrow),
875 "Nonspacing" => Ok(Self::Nonspacing),
876 "Compatibility" => Ok(Self::Compatibility),
877 "Syllable" => Ok(Self::Syllable),
878 "Ligature" => Ok(Self::Ligature),
879 "Modifier" => Ok(Self::Modifier),
880 "Spacing Combining" => Ok(Self::SpacingCombining),
881 "Emoji" => Ok(Self::Emoji),
882 "Enclosing" => Ok(Self::Enclosing),
883 "Composition" => Ok(Self::Composition),
884 "Other" => Ok(Self::Other),
885 "Lowercase" => Ok(Self::Lowercase),
886 "Uppercase" => Ok(Self::Uppercase),
887 "Smallcaps" => Ok(Self::Smallcaps),
888 "Conjunct" => Ok(Self::Conjunct),
889
890 _ => Err(s.into()),
891 }
892 }
893}
894
895impl FromStr for Script {
896 type Err = SmolStr;
897
898 fn from_str(s: &str) -> Result<Self, Self::Err> {
899 match s {
900 "adlam" => Ok(Self::Adlam),
901 "alchemical" => Ok(Self::Alchemical),
902 "arabic" => Ok(Self::Arabic),
903 "armenian" => Ok(Self::Armenian),
904 "avestan" => Ok(Self::Avestan),
905 "balinese" => Ok(Self::Balinese),
906 "bamum" => Ok(Self::Bamum),
907 "batak" => Ok(Self::Batak),
908 "bengali" => Ok(Self::Bengali),
909 "blackLetter" => Ok(Self::BlackLetter),
910 "bopomofo" => Ok(Self::Bopomofo),
911 "brahmi" => Ok(Self::Brahmi),
912 "braille" => Ok(Self::Braille),
913 "buginese" => Ok(Self::Buginese),
914 "canadian" => Ok(Self::Canadian),
915 "chakma" => Ok(Self::Chakma),
916 "cham" => Ok(Self::Cham),
917 "cherokee" => Ok(Self::Cherokee),
918 "chorasmian" => Ok(Self::Chorasmian),
919 "coptic" => Ok(Self::Coptic),
920 "cyrillic" => Ok(Self::Cyrillic),
921 "dentistry" => Ok(Self::Dentistry),
922 "deseret" => Ok(Self::Deseret),
923 "devanagari" => Ok(Self::Devanagari),
924 "divesakuru" => Ok(Self::Divesakuru),
925 "elbasan" => Ok(Self::Elbasan),
926 "elymaic" => Ok(Self::Elymaic),
927 "ethiopic" => Ok(Self::Ethiopic),
928 "georgian" => Ok(Self::Georgian),
929 "glagolitic" => Ok(Self::Glagolitic),
930 "gothic" => Ok(Self::Gothic),
931 "greek" => Ok(Self::Greek),
932 "gujarati" => Ok(Self::Gujarati),
933 "gurmukhi" => Ok(Self::Gurmukhi),
934 "han" => Ok(Self::Han),
935 "hangul" => Ok(Self::Hangul),
936 "hebrew" => Ok(Self::Hebrew),
937 "javanese" => Ok(Self::Javanese),
938 "kana" => Ok(Self::Kana),
939 "kannada" => Ok(Self::Kannada),
940 "kawi" => Ok(Self::Kawi),
941 "kayahli" => Ok(Self::Kayahli),
942 "khmer" => Ok(Self::Khmer),
943 "khojki" => Ok(Self::Khojki),
944 "lao" => Ok(Self::Lao),
945 "latin" => Ok(Self::Latin),
946 "lepcha" => Ok(Self::Lepcha),
947 "lue" => Ok(Self::Lue),
948 "mahjong" => Ok(Self::Mahjong),
949 "malayalam" => Ok(Self::Malayalam),
950 "mandaic" => Ok(Self::Mandaic),
951 "math" => Ok(Self::Math),
952 "mongolian" => Ok(Self::Mongolian),
953 "musical" => Ok(Self::Musical),
954 "myanmar" => Ok(Self::Myanmar),
955 "nko" => Ok(Self::Nko),
956 "nyiakeng puachue hmong" => Ok(Self::NyiakengPuachueHmong),
957 "ogham" => Ok(Self::Ogham),
958 "old georgian" => Ok(Self::OldGeorgian),
959 "oriya" => Ok(Self::Oriya),
960 "osage" => Ok(Self::Osage),
961 "osmanya" => Ok(Self::Osmanya),
962 "pahawh hmong" => Ok(Self::PahawhHmong),
963 "phaistosDisc" => Ok(Self::PhaistosDisc),
964 "rovas" => Ok(Self::Rovas),
965 "runic" => Ok(Self::Runic),
966 "samaritan" => Ok(Self::Samaritan),
967 "shavian" => Ok(Self::Shavian),
968 "sinhala" => Ok(Self::Sinhala),
969 "syriac" => Ok(Self::Syriac),
970 "tagalog" => Ok(Self::Tagalog),
971 "tamil" => Ok(Self::Tamil),
972 "telugu" => Ok(Self::Telugu),
973 "thaana" => Ok(Self::Thaana),
974 "thai" => Ok(Self::Thai),
975 "tham" => Ok(Self::Tham),
976 "tibet" => Ok(Self::Tibet),
977 "tifinagh" => Ok(Self::Tifinagh),
978 "tulu tigalari" => Ok(Self::TuluTigalari),
979 "vai" => Ok(Self::Vai),
980 "yi" => Ok(Self::Yi),
981 _ => Err(s.into()),
982 }
983 }
984}
985
986impl Display for Category {
987 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
988 match self {
989 Self::Mark => write!(f, "Mark"),
990 Self::Space => write!(f, "Space"),
991 Self::Separator => write!(f, "Separator"),
992 Self::Letter => write!(f, "Letter"),
993 Self::Number => write!(f, "Number"),
994 Self::Symbol => write!(f, "Symbol"),
995 Self::Punctuation => write!(f, "Punctuation"),
996 Self::Other => write!(f, "Other"),
997 }
998 }
999}
1000
1001impl Display for Subcategory {
1002 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1003 match self {
1004 Self::Spacing => write!(f, "Spacing"),
1005 Self::Radical => write!(f, "Radical"),
1006 Self::Math => write!(f, "Math"),
1007 Self::Superscript => write!(f, "Superscript"),
1008 Self::Geometry => write!(f, "Geometry"),
1009 Self::Dash => write!(f, "Dash"),
1010 Self::DecimalDigit => write!(f, "Decimal Digit"),
1011 Self::Currency => write!(f, "Currency"),
1012 Self::Fraction => write!(f, "Fraction"),
1013 Self::Halfform => write!(f, "Halfform"),
1014 Self::Small => write!(f, "Small"),
1015 Self::Number => write!(f, "Number"),
1016 Self::Quote => write!(f, "Quote"),
1017 Self::Space => write!(f, "Space"),
1018 Self::Letter => write!(f, "Letter"),
1019 Self::Jamo => write!(f, "Jamo"),
1020 Self::Format => write!(f, "Format"),
1021 Self::Parenthesis => write!(f, "Parenthesis"),
1022 Self::Matra => write!(f, "Matra"),
1023 Self::Arrow => write!(f, "Arrow"),
1024 Self::Nonspacing => write!(f, "Nonspacing"),
1025 Self::Compatibility => write!(f, "Compatibility"),
1026 Self::Syllable => write!(f, "Syllable"),
1027 Self::Ligature => write!(f, "Ligature"),
1028 Self::Modifier => write!(f, "Modifier"),
1029 Self::SpacingCombining => write!(f, "Spacing Combining"),
1030 Self::Emoji => write!(f, "Emoji"),
1031 Self::Enclosing => write!(f, "Enclosing"),
1032 Self::Composition => write!(f, "Composition"),
1033 Self::Lowercase => write!(f, "Lowercase"),
1034 Self::Uppercase => write!(f, "Uppercase"),
1035 Self::Smallcaps => write!(f, "Smallcaps"),
1036 Self::Conjunct => write!(f, "Conjunct"),
1037 Self::Other => write!(f, "Other"),
1038 }
1039 }
1040}
1041
1042#[cfg(test)]
1043mod tests {
1044
1045 use super::*;
1046 use rstest::rstest;
1047
1048 #[test]
1049 fn simple_overrides() {
1050 let overrides = HashMap::from([(
1051 "A".into(),
1052 QueryResult {
1053 category: Category::Mark,
1054 subcategory: Some(Subcategory::SpacingCombining),
1055 codepoint: Some(b'A' as u32),
1056 script: Some(Script::Alchemical),
1057 production_name: Some(ProductionName::Custom("MagicA".into())),
1058 },
1059 )]);
1060 let data = GlyphData::new(Some(overrides));
1061
1062 let result = data.query("A", None).unwrap();
1063 assert_eq!(result.category, Category::Mark);
1064 assert_eq!(result.subcategory, Some(Subcategory::SpacingCombining));
1065 assert_eq!(result.codepoint, Some(b'A' as u32));
1066 assert_eq!(result.script, Some(Script::Alchemical));
1067 assert_eq!(result.production_name, Some("MagicA".into()));
1068 }
1069
1070 #[test]
1071 fn overrides_from_file() {
1072 let data =
1073 GlyphData::with_override_file(Path::new("./data/GlyphData_override_test.xml")).unwrap();
1074 assert_eq!(data.query("zero", None).unwrap().category, Category::Other);
1075 assert_eq!(data.query("C", None).unwrap().category, Category::Number);
1076 assert_eq!(
1077 data.query("Yogh", None).unwrap().production_name,
1078 Some("Yolo".into())
1079 );
1080 }
1081
1082 #[test]
1087 fn parse_non_ligature_subcategories() {
1088 assert_eq!(Subcategory::from_str("Other"), Ok(Subcategory::Other));
1089 assert_eq!(Subcategory::from_str("Conjunct"), Ok(Subcategory::Conjunct));
1090 }
1091
1092 fn get_category(name: &str, codepoints: &[u32]) -> Option<(Category, Option<Subcategory>)> {
1093 let codepoints = codepoints.iter().copied().collect();
1094 GlyphData::new(None)
1095 .query(name, Some(&codepoints))
1096 .map(|result| (result.category, result.subcategory))
1097 }
1098
1099 #[test]
1101 fn py_test_category() {
1102 for (name, expected) in [
1103 (".notdef", Some((Category::Separator, None))),
1104 ("uni000D", Some((Category::Separator, None))),
1106 (
1107 "boxHeavyUp",
1108 Some((Category::Symbol, Some(Subcategory::Geometry))),
1109 ),
1110 ("eacute", Some((Category::Letter, None))),
1111 ("Abreveacute", Some((Category::Letter, None))),
1112 ("C-fraktur", Some((Category::Letter, None))),
1113 ("fi", Some((Category::Letter, Some(Subcategory::Ligature)))),
1114 (
1115 "fi.alt",
1116 Some((Category::Letter, Some(Subcategory::Ligature))),
1117 ),
1118 (
1119 "hib-ko",
1120 Some((Category::Letter, Some(Subcategory::Syllable))),
1121 ),
1122 (
1123 "one.foo",
1124 Some((Category::Number, Some(Subcategory::DecimalDigit))),
1125 ),
1126 (
1127 "one_two.foo",
1128 Some((Category::Number, Some(Subcategory::Ligature))),
1129 ),
1130 (
1131 "o_f_f_i",
1132 Some((Category::Letter, Some(Subcategory::Ligature))),
1133 ),
1134 (
1135 "o_f_f_i.foo",
1136 Some((Category::Letter, Some(Subcategory::Ligature))),
1137 ),
1138 (
1139 "ain_alefMaksura-ar.fina",
1140 Some((Category::Letter, Some(Subcategory::Ligature))),
1141 ),
1142 (
1143 "brevecomb",
1144 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1145 ),
1146 (
1147 "brevecomb.case",
1148 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1149 ),
1150 (
1151 "brevecomb_acutecomb",
1152 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1153 ),
1154 (
1155 "brevecomb_acutecomb.case",
1156 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1157 ),
1158 (
1159 "caroncomb_dotaccentcomb",
1160 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1161 ),
1162 (
1163 "dieresiscomb_caroncomb",
1164 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1165 ),
1166 (
1167 "dieresiscomb_macroncomb",
1168 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1169 ),
1170 (
1171 "dotaccentcomb_macroncomb",
1172 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1173 ),
1174 (
1175 "macroncomb_dieresiscomb",
1176 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1177 ),
1178 (
1179 "dotaccentcomb_o",
1180 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1181 ),
1182 (
1183 "macronlowmod_O",
1184 Some((Category::Mark, Some(Subcategory::Modifier))),
1185 ),
1186 ("O_o", Some((Category::Letter, Some(Subcategory::Ligature)))),
1187 (
1188 "O_dotaccentcomb_o",
1189 Some((Category::Letter, Some(Subcategory::Ligature))),
1190 ),
1191 ("O_dotaccentcomb", Some((Category::Letter, None))),
1192 (
1193 "O_period",
1194 Some((Category::Letter, Some(Subcategory::Ligature))),
1195 ),
1196 ("O_nbspace", Some((Category::Letter, None))),
1197 ("_a", None),
1198 ("_aaa", None),
1199 (
1200 "dal_alef-ar",
1201 Some((Category::Letter, Some(Subcategory::Ligature))),
1202 ),
1203 (
1204 "dal_lam-ar.dlig",
1205 Some((Category::Letter, Some(Subcategory::Ligature))),
1206 ),
1207 ("po-khmer", Some((Category::Letter, None))),
1208 (
1209 "po-khmer.below",
1210 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1211 ),
1212 (
1213 "po-khmer.below.ro",
1214 Some((Category::Mark, Some(Subcategory::Nonspacing))),
1215 ),
1216 ] {
1217 let result = get_category(name, &[]);
1218 assert_eq!(result, expected, "{name}: {result:?} != {expected:?}");
1219 }
1220 }
1221
1222 #[test]
1224 fn py_category_by_unicode() {
1225 let result = get_category("SignU.bn", &[0x09C1]);
1228 assert_eq!(
1229 result,
1230 Some((Category::Mark, Some(Subcategory::Nonspacing)))
1231 )
1232 }
1233
1234 #[test]
1237 fn py_bug_232() {
1238 let u = get_category("uni07F0", &[]);
1239 assert_eq!(u, Some((Category::Mark, Some(Subcategory::Nonspacing))));
1240 let g = get_category("longlowtonecomb-nko", &[]);
1241 assert_eq!(g, Some((Category::Mark, Some(Subcategory::Nonspacing))));
1242 }
1243
1244 #[test]
1245 fn unknown_name_combined_with_mark() {
1246 assert_eq!(get_category("Whata-WEIRDNameLOL_brevecomb", &[]), None)
1248 }
1249
1250 #[test]
1251 fn known_name_with_unknown_mark() {
1252 assert_eq!(
1255 get_category("i_acutecombcombcy", &[]),
1256 get_category("i", &[])
1257 )
1258 }
1259
1260 #[test]
1261 fn match_prod_name_with_suffix() {
1262 assert_eq!(
1265 Some((Category::Letter, None)),
1266 get_category("uni17BF.b", &[]),
1267 )
1268 }
1269
1270 #[rstest(name, expected,
1271 case("A", None), case("z", None),
1273 case("nbspace", Some("uni00A0")),
1274 case("nonbreakingspace", Some("uni00A0")), case("uni00A0", Some("uni00A0")), case("guillemetleft", Some("guillemotleft")),
1279 case("twosevenths", Some("two_fraction_seven")),
1280 case("idotaccent", Some("i.loclTRK")),
1281 case("idotless", Some("dotlessi")),
1282 case("Jacute", Some("uni004A0301")),
1283 case("scurl", Some("u1DF1E")),
1284 case("Delta", Some("uni0394")),
1287 case("increment", Some("uni2206")),
1288 case("dog-ko", Some("uniB3C5")),
1289 case("bau-kannada", Some("uni0CAC0CCC")),
1290 case("EnglandFlag", Some("u1F3F4E0067E0062E0065E006EE0067E007F")),
1291 case("pileOfPoo", Some("u1F4A9")),
1292 case("lam_alef-ar.fina", Some("uni06440627.fina")),
1293 )]
1294 fn query_production_names(name: &str, expected: Option<&str>) {
1295 let production_name = GlyphData::new(None)
1296 .query_no_synthesis(name, None)
1297 .unwrap()
1298 .production_name
1299 .map(|p| p.to_string());
1300 assert_eq!(
1301 production_name,
1302 expected.map(Into::into),
1303 "{name}: {production_name:?} != {expected:?}"
1304 );
1305 }
1306
1307 #[rstest(
1311 name,
1312 expected,
1313 case("Ech_Vew-arm.liga", "uni0535054E.liga"),
1314 case("aiMatra_anusvara-deva", "uni09480902"),
1315 case("aiMatra_reph_anusvara-deva", "uni09480930094D0902"),
1316 case("ca_iMatra-tamil", "uni0B9A0BBF"),
1317 case("ch_ya-deva", "uni091B094D092F"),
1318 case("d_dh_ya-deva", "uni0926094D0927094D092F"),
1319 case("da-khmer.below.ro", "uni17D2178A.ro"),
1320 case("da_rVocalicMatra-deva", "uni09260943"),
1321 case("dd_dda-deva", "uni0921094D0921"),
1322 case("eShortMatra_reph_anusvara-deva", "uni09460930094D0902"),
1323 case("ech_vew-arm.liga.sc", "uni0565057E.liga.sc"),
1324 case("finalkaf_qamats-hb", "uni05DA05B8"),
1325 case("finalkaf_sheva-hb", "uni05DA05B0"),
1326 case("finalkafdagesh_qamats-hb", "uniFB3A05B8"),
1327 case("finalkafdagesh_sheva-hb", "uniFB3A05B0"),
1328 case("h_la-deva", "uni0939094D0932"),
1329 case("ha_iMatra-tamil", "uni0BB90BBF"),
1330 case("hatafpatah_siluqleft-hb", "uni05B205BD"),
1331 case("iMark_toandakhiat-khmer.narrow", "uni17B717CD.narrow"),
1332 case("idotaccent.sc", "i.loclTRK.sc"),
1333 case("iiMatra_reph-deva", "uni09400930094D"),
1334 case("iiMatra_reph-deva.alt2", "uni09400930094D.alt2"),
1335 case("j_ny-deva", "uni091C094D091E094D"),
1336 case("j_ny-deva.alt2", "uni091C094D091E094D.alt2"),
1337 case("mo-khmer.below.ro", "uni17D21798.ro"),
1338 case("moMa_underscore-thai", "uni0E21005F"),
1339 case("nno-khmer.below.narrow1", "uni17D2178E.narrow1"),
1340 case("nyo-khmer.full.below.narrow", "uni17D21789.full.below.narrow"),
1341 case("sh_ra_iiMatra-tamil", "uni0BB60BCD0BB00BC0"),
1342 case("A_A", "A_A"),
1344 case("a_a.sc", "a_a.sc"),
1345 case("brevecomb_acutecomb", "uni03060301"),
1346 case("brevecomb_acutecomb.case", "uni03060301.case"),
1347 case("pileOfPoo_pileOfPoo", "u1F4A9_u1F4A9"),
1348 case("pileOfPoo.ss01", "u1F4A9.ss01"),
1349 case("lam_alef-ar.fina.ss02", "uni06440627.fina.ss02"),
1350 case("one-han.half.vert", "uni3021.half.vert"),
1353 case("one_parenleft-han.vert", "uni30210028.vert"),
1354 case("one_one-han.paren", "uni30213021.paren"),
1355 case("one_parenleftsmall-han.vert", "uni3021FE59.vert"),
1356 )]
1357 fn synthetic_production_names(name: &str, expected: &str) {
1358 let production_name = GlyphData::new(None)
1359 .query(name, None)
1360 .unwrap()
1361 .production_name
1362 .unwrap()
1363 .to_string();
1364 assert_eq!(
1365 &production_name, expected,
1366 "{name}: {production_name:?} != {expected:?}"
1367 );
1368 }
1369
1370 #[test]
1372 fn prod_name_lookup_before_codepoint_lookup() {
1373 let codepoints: BTreeSet<u32> = [0x25BA, 0x25B6].into_iter().collect();
1378 let result = GlyphData::new(None)
1379 .query("triagrt", Some(&codepoints))
1380 .unwrap();
1381 assert_eq!(
1384 result.production_name.as_ref().map(|p| p.to_string()),
1385 Some("triagrt".to_string()),
1386 );
1387 }
1388}