1use crate::line_index::LineIndex;
2use crate::link_flags::LinkFlags;
3use crate::link_network::{LinkId, LinkMetadata, LinkNetwork, LinkType};
4use crate::source::{ByteRange, SourceSpan};
5
6#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8pub struct NaturalLanguageGrammarFixture {
9 language: &'static str,
10 grammatical_source: &'static str,
11 ungrammatical_source: &'static str,
12 provenance: &'static str,
13}
14
15impl NaturalLanguageGrammarFixture {
16 #[must_use]
18 pub const fn language(&self) -> &'static str {
19 self.language
20 }
21
22 #[must_use]
24 pub const fn grammatical_source(&self) -> &'static str {
25 self.grammatical_source
26 }
27
28 #[must_use]
30 pub const fn ungrammatical_source(&self) -> &'static str {
31 self.ungrammatical_source
32 }
33
34 #[must_use]
36 pub const fn provenance(&self) -> &'static str {
37 self.provenance
38 }
39}
40
41const STARTER_GRAMMAR_PROVENANCE: &str =
42 "repo-authored starter pass/fail sentence; license: Unlicense; \
43 morphosyntax tag names use Universal Dependencies v2 UPOS/UFeats/deprel vocabulary; \
44 no UD treebank sentence data imported";
45
46pub const NATURAL_LANGUAGE_GRAMMAR_FIXTURES: &[NaturalLanguageGrammarFixture] = &[
48 NaturalLanguageGrammarFixture {
49 language: "English",
50 grammatical_source: "Hawaii is a state.\n",
51 ungrammatical_source: "Hawaii are a state.\n",
52 provenance: STARTER_GRAMMAR_PROVENANCE,
53 },
54 NaturalLanguageGrammarFixture {
55 language: "Mandarin Chinese",
56 grammatical_source: "你好。\n",
57 ungrammatical_source: "你好 的。\n",
58 provenance: STARTER_GRAMMAR_PROVENANCE,
59 },
60 NaturalLanguageGrammarFixture {
61 language: "Hindi",
62 grammatical_source: "नमस्ते।\n",
63 ungrammatical_source: "नमस्ते है।\n",
64 provenance: STARTER_GRAMMAR_PROVENANCE,
65 },
66 NaturalLanguageGrammarFixture {
67 language: "Spanish",
68 grammatical_source: "Hawaii es un estado.\n",
69 ungrammatical_source: "Hawaii son un estado.\n",
70 provenance: STARTER_GRAMMAR_PROVENANCE,
71 },
72 NaturalLanguageGrammarFixture {
73 language: "Modern Standard Arabic",
74 grammatical_source: "مرحبا.\n",
75 ungrammatical_source: "مرحبا هو.\n",
76 provenance: STARTER_GRAMMAR_PROVENANCE,
77 },
78 NaturalLanguageGrammarFixture {
79 language: "French",
80 grammatical_source: "Hawaii est un etat.\n",
81 ungrammatical_source: "Hawaii sont un etat.\n",
82 provenance: STARTER_GRAMMAR_PROVENANCE,
83 },
84 NaturalLanguageGrammarFixture {
85 language: "Bengali",
86 grammatical_source: "নমস্কার।\n",
87 ungrammatical_source: "নমস্কার আছে।\n",
88 provenance: STARTER_GRAMMAR_PROVENANCE,
89 },
90 NaturalLanguageGrammarFixture {
91 language: "Portuguese",
92 grammatical_source: "Hawaii e um estado.\n",
93 ungrammatical_source: "Hawaii sao um estado.\n",
94 provenance: STARTER_GRAMMAR_PROVENANCE,
95 },
96 NaturalLanguageGrammarFixture {
97 language: "Russian",
98 grammatical_source: "Гавайи это штат.\n",
99 ungrammatical_source: "Гавайи это штаты.\n",
100 provenance: STARTER_GRAMMAR_PROVENANCE,
101 },
102 NaturalLanguageGrammarFixture {
103 language: "Urdu",
104 grammatical_source: "سلام۔\n",
105 ungrammatical_source: "سلام ہے۔\n",
106 provenance: STARTER_GRAMMAR_PROVENANCE,
107 },
108];
109
110#[derive(Clone, Debug, PartialEq, Eq)]
111struct GrammarToken {
112 text: String,
113 range: ByteRange,
114}
115
116#[derive(Clone, Copy, Debug, PartialEq, Eq)]
117struct MorphAnalysis {
118 upos: &'static str,
119 features: &'static [&'static str],
120 deprel: &'static str,
121}
122
123#[derive(Clone, Copy, Debug, PartialEq, Eq)]
124struct LexiconEntry {
125 language: &'static str,
126 surface: &'static str,
127 analysis: MorphAnalysis,
128}
129
130#[derive(Clone, Copy, Debug, PartialEq, Eq)]
131struct SentenceGrammar {
132 language: &'static str,
133 accepted: &'static [&'static str],
134}
135
136const NO_FEATURES: &[&str] = &[];
137const NUMBER_SING: &[&str] = &["Number=Sing"];
138const NUMBER_PLUR: &[&str] = &["Number=Plur"];
139const INDEFINITE_ARTICLE: &[&str] = &["Definite=Ind", "PronType=Art"];
140const AUX_SINGULAR: &[&str] = &[
141 "Mood=Ind",
142 "Number=Sing",
143 "Person=3",
144 "Tense=Pres",
145 "VerbForm=Fin",
146];
147const AUX_PLURAL: &[&str] = &[
148 "Mood=Ind",
149 "Number=Plur",
150 "Person=3",
151 "Tense=Pres",
152 "VerbForm=Fin",
153];
154const PRON_SINGULAR: &[&str] = &["Number=Sing", "Person=3", "PronType=Prs"];
155
156const PUNCT_ANALYSIS: MorphAnalysis = MorphAnalysis {
157 upos: "PUNCT",
158 features: NO_FEATURES,
159 deprel: "punct",
160};
161
162const SENTENCE_GRAMMARS: &[SentenceGrammar] = &[
163 SentenceGrammar {
164 language: "English",
165 accepted: &["Hawaii", "is", "a", "state", "."],
166 },
167 SentenceGrammar {
168 language: "Mandarin Chinese",
169 accepted: &["你好", "\u{3002}"],
170 },
171 SentenceGrammar {
172 language: "Hindi",
173 accepted: &["नमस्ते", "\u{0964}"],
174 },
175 SentenceGrammar {
176 language: "Spanish",
177 accepted: &["Hawaii", "es", "un", "estado", "."],
178 },
179 SentenceGrammar {
180 language: "Modern Standard Arabic",
181 accepted: &["مرحبا", "."],
182 },
183 SentenceGrammar {
184 language: "French",
185 accepted: &["Hawaii", "est", "un", "etat", "."],
186 },
187 SentenceGrammar {
188 language: "Bengali",
189 accepted: &["নমস্কার", "\u{0964}"],
190 },
191 SentenceGrammar {
192 language: "Portuguese",
193 accepted: &["Hawaii", "e", "um", "estado", "."],
194 },
195 SentenceGrammar {
196 language: "Russian",
197 accepted: &["Гавайи", "это", "штат", "."],
198 },
199 SentenceGrammar {
200 language: "Urdu",
201 accepted: &["سلام", "\u{06d4}"],
202 },
203];
204
205const LEXICON: &[LexiconEntry] = &[
206 LexiconEntry {
207 language: "English",
208 surface: "Hawaii",
209 analysis: MorphAnalysis {
210 upos: "PROPN",
211 features: NUMBER_SING,
212 deprel: "nsubj",
213 },
214 },
215 LexiconEntry {
216 language: "English",
217 surface: "is",
218 analysis: MorphAnalysis {
219 upos: "AUX",
220 features: AUX_SINGULAR,
221 deprel: "cop",
222 },
223 },
224 LexiconEntry {
225 language: "English",
226 surface: "are",
227 analysis: MorphAnalysis {
228 upos: "AUX",
229 features: AUX_PLURAL,
230 deprel: "cop",
231 },
232 },
233 LexiconEntry {
234 language: "English",
235 surface: "a",
236 analysis: MorphAnalysis {
237 upos: "DET",
238 features: INDEFINITE_ARTICLE,
239 deprel: "det",
240 },
241 },
242 LexiconEntry {
243 language: "English",
244 surface: "state",
245 analysis: MorphAnalysis {
246 upos: "NOUN",
247 features: NUMBER_SING,
248 deprel: "root",
249 },
250 },
251 LexiconEntry {
252 language: "Mandarin Chinese",
253 surface: "你好",
254 analysis: MorphAnalysis {
255 upos: "INTJ",
256 features: NO_FEATURES,
257 deprel: "root",
258 },
259 },
260 LexiconEntry {
261 language: "Mandarin Chinese",
262 surface: "的",
263 analysis: MorphAnalysis {
264 upos: "PART",
265 features: NO_FEATURES,
266 deprel: "mark",
267 },
268 },
269 LexiconEntry {
270 language: "Hindi",
271 surface: "नमस्ते",
272 analysis: MorphAnalysis {
273 upos: "INTJ",
274 features: NO_FEATURES,
275 deprel: "root",
276 },
277 },
278 LexiconEntry {
279 language: "Hindi",
280 surface: "है",
281 analysis: MorphAnalysis {
282 upos: "AUX",
283 features: AUX_SINGULAR,
284 deprel: "cop",
285 },
286 },
287 LexiconEntry {
288 language: "Spanish",
289 surface: "Hawaii",
290 analysis: MorphAnalysis {
291 upos: "PROPN",
292 features: NUMBER_SING,
293 deprel: "nsubj",
294 },
295 },
296 LexiconEntry {
297 language: "Spanish",
298 surface: "es",
299 analysis: MorphAnalysis {
300 upos: "AUX",
301 features: AUX_SINGULAR,
302 deprel: "cop",
303 },
304 },
305 LexiconEntry {
306 language: "Spanish",
307 surface: "son",
308 analysis: MorphAnalysis {
309 upos: "AUX",
310 features: AUX_PLURAL,
311 deprel: "cop",
312 },
313 },
314 LexiconEntry {
315 language: "Spanish",
316 surface: "un",
317 analysis: MorphAnalysis {
318 upos: "DET",
319 features: INDEFINITE_ARTICLE,
320 deprel: "det",
321 },
322 },
323 LexiconEntry {
324 language: "Spanish",
325 surface: "estado",
326 analysis: MorphAnalysis {
327 upos: "NOUN",
328 features: NUMBER_SING,
329 deprel: "root",
330 },
331 },
332 LexiconEntry {
333 language: "Modern Standard Arabic",
334 surface: "مرحبا",
335 analysis: MorphAnalysis {
336 upos: "INTJ",
337 features: NO_FEATURES,
338 deprel: "root",
339 },
340 },
341 LexiconEntry {
342 language: "Modern Standard Arabic",
343 surface: "هو",
344 analysis: MorphAnalysis {
345 upos: "PRON",
346 features: PRON_SINGULAR,
347 deprel: "dep",
348 },
349 },
350 LexiconEntry {
351 language: "French",
352 surface: "Hawaii",
353 analysis: MorphAnalysis {
354 upos: "PROPN",
355 features: NUMBER_SING,
356 deprel: "nsubj",
357 },
358 },
359 LexiconEntry {
360 language: "French",
361 surface: "est",
362 analysis: MorphAnalysis {
363 upos: "AUX",
364 features: AUX_SINGULAR,
365 deprel: "cop",
366 },
367 },
368 LexiconEntry {
369 language: "French",
370 surface: "sont",
371 analysis: MorphAnalysis {
372 upos: "AUX",
373 features: AUX_PLURAL,
374 deprel: "cop",
375 },
376 },
377 LexiconEntry {
378 language: "French",
379 surface: "un",
380 analysis: MorphAnalysis {
381 upos: "DET",
382 features: INDEFINITE_ARTICLE,
383 deprel: "det",
384 },
385 },
386 LexiconEntry {
387 language: "French",
388 surface: "etat",
389 analysis: MorphAnalysis {
390 upos: "NOUN",
391 features: NUMBER_SING,
392 deprel: "root",
393 },
394 },
395 LexiconEntry {
396 language: "Bengali",
397 surface: "নমস্কার",
398 analysis: MorphAnalysis {
399 upos: "INTJ",
400 features: NO_FEATURES,
401 deprel: "root",
402 },
403 },
404 LexiconEntry {
405 language: "Bengali",
406 surface: "আছে",
407 analysis: MorphAnalysis {
408 upos: "AUX",
409 features: AUX_SINGULAR,
410 deprel: "cop",
411 },
412 },
413 LexiconEntry {
414 language: "Portuguese",
415 surface: "Hawaii",
416 analysis: MorphAnalysis {
417 upos: "PROPN",
418 features: NUMBER_SING,
419 deprel: "nsubj",
420 },
421 },
422 LexiconEntry {
423 language: "Portuguese",
424 surface: "e",
425 analysis: MorphAnalysis {
426 upos: "AUX",
427 features: AUX_SINGULAR,
428 deprel: "cop",
429 },
430 },
431 LexiconEntry {
432 language: "Portuguese",
433 surface: "sao",
434 analysis: MorphAnalysis {
435 upos: "AUX",
436 features: AUX_PLURAL,
437 deprel: "cop",
438 },
439 },
440 LexiconEntry {
441 language: "Portuguese",
442 surface: "um",
443 analysis: MorphAnalysis {
444 upos: "DET",
445 features: INDEFINITE_ARTICLE,
446 deprel: "det",
447 },
448 },
449 LexiconEntry {
450 language: "Portuguese",
451 surface: "estado",
452 analysis: MorphAnalysis {
453 upos: "NOUN",
454 features: NUMBER_SING,
455 deprel: "root",
456 },
457 },
458 LexiconEntry {
459 language: "Russian",
460 surface: "Гавайи",
461 analysis: MorphAnalysis {
462 upos: "PROPN",
463 features: NUMBER_SING,
464 deprel: "nsubj",
465 },
466 },
467 LexiconEntry {
468 language: "Russian",
469 surface: "это",
470 analysis: MorphAnalysis {
471 upos: "PRON",
472 features: PRON_SINGULAR,
473 deprel: "cop",
474 },
475 },
476 LexiconEntry {
477 language: "Russian",
478 surface: "штат",
479 analysis: MorphAnalysis {
480 upos: "NOUN",
481 features: NUMBER_SING,
482 deprel: "root",
483 },
484 },
485 LexiconEntry {
486 language: "Russian",
487 surface: "штаты",
488 analysis: MorphAnalysis {
489 upos: "NOUN",
490 features: NUMBER_PLUR,
491 deprel: "root",
492 },
493 },
494 LexiconEntry {
495 language: "Urdu",
496 surface: "سلام",
497 analysis: MorphAnalysis {
498 upos: "INTJ",
499 features: NO_FEATURES,
500 deprel: "root",
501 },
502 },
503 LexiconEntry {
504 language: "Urdu",
505 surface: "ہے",
506 analysis: MorphAnalysis {
507 upos: "AUX",
508 features: AUX_SINGULAR,
509 deprel: "cop",
510 },
511 },
512];
513
514pub fn annotate_morphosyntax(
515 network: &mut LinkNetwork,
516 region: LinkId,
517 text: &str,
518 lines: &LineIndex,
519 language: &str,
520 span: SourceSpan,
521) {
522 let grammar_tokens = grammar_tokens(text);
523 if grammar_tokens.is_empty() {
524 return;
525 }
526
527 let sentence_is_accepted = sentence_grammar(language)
528 .is_some_and(|grammar| token_surfaces_match(&grammar_tokens, grammar.accepted));
529 let should_report_errors =
530 is_registered_grammar_fixture(language, text) && !sentence_is_accepted;
531 let sentence_flags = if sentence_is_accepted {
532 LinkFlags::clean()
533 } else if should_report_errors {
534 LinkFlags::containing_error()
535 } else {
536 LinkFlags::clean()
537 };
538 let sentence = network.insert_link(
539 [region],
540 LinkMetadata::new()
541 .with_link_type(LinkType::Syntax)
542 .with_named(true)
543 .with_term("natural-language:sentence")
544 .with_language(language)
545 .with_span(span)
546 .with_flags(sentence_flags),
547 );
548
549 for token in &grammar_tokens {
550 let token_span = span_for_range(lines, token.range.start(), token.range.end());
551 let form = network.insert_link(
552 [sentence],
553 LinkMetadata::new()
554 .with_link_type(LinkType::Syntax)
555 .with_named(true)
556 .with_term(format!("form:{}", token.text))
557 .with_language(language)
558 .with_span(token_span),
559 );
560
561 if let Some(analysis) = morphology_for(language, &token.text) {
562 insert_upos_link(network, form, language, token_span, analysis.upos);
563 for feature in analysis.features {
564 insert_ufeat_link(network, form, language, token_span, feature);
565 }
566 insert_deprel_link(
567 network,
568 sentence,
569 form,
570 language,
571 token_span,
572 analysis.deprel,
573 );
574 } else if should_report_errors {
575 insert_error_link(
576 network,
577 [sentence, form],
578 "natural-language:error:unknown-token",
579 language,
580 token_span,
581 );
582 }
583 }
584
585 if should_report_errors {
586 insert_error_link(
587 network,
588 [sentence],
589 "natural-language:error:grammar",
590 language,
591 span,
592 );
593 }
594}
595
596fn insert_upos_link(
597 network: &mut LinkNetwork,
598 form: LinkId,
599 language: &str,
600 span: SourceSpan,
601 upos: &str,
602) -> LinkId {
603 network.insert_link(
604 [form],
605 LinkMetadata::new()
606 .with_link_type(LinkType::Syntax)
607 .with_named(true)
608 .with_term(format!("upos:{upos}"))
609 .with_language(language)
610 .with_span(span),
611 )
612}
613
614fn insert_ufeat_link(
615 network: &mut LinkNetwork,
616 form: LinkId,
617 language: &str,
618 span: SourceSpan,
619 feature: &str,
620) -> LinkId {
621 network.insert_link(
622 [form],
623 LinkMetadata::new()
624 .with_link_type(LinkType::Syntax)
625 .with_named(true)
626 .with_term(format!("ufeat:{feature}"))
627 .with_language(language)
628 .with_span(span),
629 )
630}
631
632fn insert_deprel_link(
633 network: &mut LinkNetwork,
634 sentence: LinkId,
635 form: LinkId,
636 language: &str,
637 span: SourceSpan,
638 deprel: &str,
639) -> LinkId {
640 network.insert_link(
641 [sentence, form],
642 LinkMetadata::new()
643 .with_link_type(LinkType::Syntax)
644 .with_named(true)
645 .with_term(format!("deprel:{deprel}"))
646 .with_language(language)
647 .with_span(span),
648 )
649}
650
651fn insert_error_link<const N: usize>(
652 network: &mut LinkNetwork,
653 references: [LinkId; N],
654 term: &'static str,
655 language: &str,
656 span: SourceSpan,
657) -> LinkId {
658 network.insert_link(
659 references,
660 LinkMetadata::new()
661 .with_link_type(LinkType::Syntax)
662 .with_named(true)
663 .with_term(term)
664 .with_language(language)
665 .with_span(span)
666 .with_flags(LinkFlags::error()),
667 )
668}
669
670fn sentence_grammar(language: &str) -> Option<SentenceGrammar> {
671 SENTENCE_GRAMMARS
672 .iter()
673 .find(|grammar| grammar.language == language)
674 .copied()
675}
676
677fn token_surfaces_match(tokens: &[GrammarToken], expected: &[&str]) -> bool {
678 tokens.len() == expected.len()
679 && tokens
680 .iter()
681 .zip(expected.iter().copied())
682 .all(|(token, expected)| token.text == expected)
683}
684
685fn morphology_for(language: &str, surface: &str) -> Option<MorphAnalysis> {
686 if is_sentence_punctuation_token(surface) {
687 return Some(PUNCT_ANALYSIS);
688 }
689
690 LEXICON
691 .iter()
692 .find(|entry| entry.language == language && entry.surface == surface)
693 .map(|entry| entry.analysis)
694}
695
696fn is_registered_grammar_fixture(language: &str, text: &str) -> bool {
697 NATURAL_LANGUAGE_GRAMMAR_FIXTURES
698 .iter()
699 .filter(|fixture| fixture.language == language)
700 .any(|fixture| text == fixture.grammatical_source || text == fixture.ungrammatical_source)
701}
702
703fn grammar_tokens(text: &str) -> Vec<GrammarToken> {
704 let mut tokens = Vec::new();
705 let mut word_start = None;
706
707 for (index, character) in text.char_indices() {
708 if character.is_whitespace() {
709 push_pending_word_token(&mut tokens, text, &mut word_start, index);
710 } else if is_sentence_punctuation(character) {
711 push_pending_word_token(&mut tokens, text, &mut word_start, index);
712 tokens.push(GrammarToken {
713 text: character.to_string(),
714 range: ByteRange::new(index, index + character.len_utf8()),
715 });
716 } else if word_start.is_none() {
717 word_start = Some(index);
718 }
719 }
720
721 push_pending_word_token(&mut tokens, text, &mut word_start, text.len());
722 tokens
723}
724
725fn push_pending_word_token(
726 tokens: &mut Vec<GrammarToken>,
727 text: &str,
728 word_start: &mut Option<usize>,
729 end: usize,
730) {
731 let Some(start) = word_start.take() else {
732 return;
733 };
734 if start == end {
735 return;
736 }
737 tokens.push(GrammarToken {
738 text: text[start..end].to_string(),
739 range: ByteRange::new(start, end),
740 });
741}
742
743fn is_sentence_punctuation_token(surface: &str) -> bool {
744 let mut characters = surface.chars();
745 let Some(character) = characters.next() else {
746 return false;
747 };
748 characters.next().is_none() && is_sentence_punctuation(character)
749}
750
751const fn is_sentence_punctuation(character: char) -> bool {
752 matches!(
753 character,
754 '.' | '!' | '?' | '\u{0964}' | '\u{3002}' | '\u{06d4}'
755 )
756}
757
758fn span_for_range(lines: &LineIndex, start: usize, end: usize) -> SourceSpan {
759 SourceSpan::new(
760 ByteRange::new(start, end),
761 lines.char_point(start),
762 lines.char_point(end),
763 )
764}