{
"case_count": 119,
"cases": [
{
"id": "ip-newyork",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
"New York"
],
"b": [
"New",
"York"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "ip-accent-gorge",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
"à",
"la",
"gorge"
],
"b": [
"a",
"la",
"gorge"
],
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "ip-abc",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
"å",
"BC"
],
"b": [
"abc"
],
"expected": {
"a2b": [
[
0
],
[
0
]
],
"b2a": [
[
0,
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "ip-noisy",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
"fあo①が",
"bar"
],
"b": [
"fあo1かb",
"ar"
],
"expected": {
"a2b": [
[
0
],
[
0,
1
]
],
"b2a": [
[
0,
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "ip-apos-upper",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
"A'B"
],
"b": [
"A",
"B"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "ip-apos-mixed",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
"A'b"
],
"b": [
"a",
"b"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "ip-empty-tokens",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
""
],
"b": [
"",
""
],
"expected": {
"a2b": [
[]
],
"b2a": [
[],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "ip-foo",
"group": "incumbent-parity",
"api": "get_alignments",
"normalizer": "default",
"a": [
"f",
"o",
"o"
],
"b": [
"fo",
"o"
],
"expected": {
"a2b": [
[
0
],
[
0
],
[
1
]
],
"b2a": [
[
0,
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-bar",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "bar",
"b": "bår",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-aring",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "å",
"b": "å",
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-kana-dakuten",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "あがさ",
"b": "あかさ",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-empty-a",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "",
"b": "a",
"expected": {
"a2b": [],
"b2a": [
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-empty-both",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "",
"b": "",
"expected": {
"a2b": [],
"b2a": []
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-tab-aring",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "å\tb",
"b": "a b",
"expected": {
"a2b": [
[
0
],
[],
[
2
]
],
"b2a": [
[
0
],
[],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-tab",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "a\tb",
"b": "a b",
"expected": {
"a2b": [
[
0
],
[],
[
2
]
],
"b2a": [
[
0
],
[],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-fullwidth",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "２０００",
"b": "2000",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
],
[
3
]
],
"b2a": [
[
0
],
[
1
],
[
2
],
[
3
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-diaeresis",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "¨",
"b": "",
"expected": {
"a2b": [
[]
],
"b2a": []
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-quotes",
"group": "incumbent-parity",
"api": "get_charmap",
"normalizer": "default",
"a": "hello``world``",
"b": "Hello \"world\"",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
],
[
3
],
[
4
],
[],
[],
[
7
],
[
8
],
[
9
],
[
10
],
[
11
],
[],
[]
],
"b2a": [
[
0
],
[
1
],
[
2
],
[
3
],
[
4
],
[],
[],
[
7
],
[
8
],
[
9
],
[
10
],
[
11
],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-basic",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"hashtag"
],
"b": [
"hash",
"##tag"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-multi",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"unbelievable",
"!"
],
"b": [
"un",
"##believ",
"##able",
"!"
],
"expected": {
"a2b": [
[
0,
1,
2
],
[
3
]
],
"b2a": [
[
0
],
[
0
],
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-cls-sep",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"look",
"up"
],
"b": [
"[CLS]",
"look",
"up",
"[SEP]"
],
"expected": {
"a2b": [
[
1
],
[
2
]
],
"b2a": [
[],
[
0
],
[
1
],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-cls-steal-l",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"look"
],
"b": [
"[CLS]",
"look",
"[SEP]"
],
"expected": {
"a2b": [
[
1
]
],
"b2a": [
[],
[
0
],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "prior behavior relies on tie-break luck, the l of [CLS] competes"
},
{
"id": "wp-sep-steal-s",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"apes"
],
"b": [
"[CLS]",
"apes",
"[SEP]"
],
"expected": {
"a2b": [
[
1
]
],
"b2a": [
[],
[
0
],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "unk-luck",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Zürich",
"is"
],
"b": [
"[UNK]",
"is"
],
"expected": {
"a2b": [
[],
[
1
]
],
"b2a": [
[],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "prior behavior maps Zurich to [UNK] via the accidental shared u, contract maps [UNK] to nothing"
},
{
"id": "wp-uncased-accent",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Café"
],
"b": [
"cafe"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-collision",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"##",
"hi"
],
"b": [
"#",
"#",
"hi"
],
"expected": {
"a2b": [
[],
[
2
]
],
"b2a": [
[],
[],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "literal ## token normalizes to empty under the default, hook escape"
},
{
"id": "wp-only-marker",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"ab"
],
"b": [
"ab",
"##"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-case",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"McDonald"
],
"b": [
"mc",
"##donald"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-numeric",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"1234"
],
"b": [
"12",
"##34"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "wp-mask",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"The",
"cat"
],
"b": [
"[CLS]",
"the",
"[MASK]",
"[SEP]"
],
"expected": {
"a2b": [
[
1
],
[]
],
"b2a": [
[],
[
0
],
[],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "prior behavior maps cat to [MASK] via the accidental shared a, contract maps [MASK] to nothing"
},
{
"id": "wp-long-word",
"group": "wordpiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"electroencephalography"
],
"b": [
"electro",
"##ence",
"##phal",
"##ograph",
"##y"
],
"expected": {
"a2b": [
[
0,
1,
2,
3,
4
]
],
"b2a": [
[
0
],
[
0
],
[
0
],
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-basic",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"New",
"York"
],
"b": [
"▁New",
"▁Y",
"ork"
],
"expected": {
"a2b": [
[
0
],
[
1,
2
]
],
"b2a": [
[
0
],
[
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-standalone",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"gate",
"2024"
],
"b": [
"▁gate",
"▁",
"2024"
],
"expected": {
"a2b": [
[
0
],
[
2
]
],
"b2a": [
[
0
],
[],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-midtoken",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"New York"
],
"b": [
"▁New▁York"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-cjk",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"東京",
"タワー"
],
"b": [
"▁",
"東京",
"タワー"
],
"expected": {
"a2b": [
[
1
],
[
2
]
],
"b2a": [
[],
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-mixed",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Hello",
"world",
"!"
],
"b": [
"▁Hello",
"▁world",
"!"
],
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-unk",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Hello",
"😀"
],
"b": [
"▁Hello",
"▁",
"<unk>"
],
"expected": {
"a2b": [
[
0
],
[]
],
"b2a": [
[
0
],
[],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-num-split",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"2024"
],
"b": [
"▁2",
"0",
"2",
"4"
],
"expected": {
"a2b": [
[
0,
1,
2,
3
]
],
"b2a": [
[
0
],
[
0
],
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-eow",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"going",
"home"
],
"b": [
"going</w>",
"home</w>"
],
"expected": {
"a2b": [
[
0
],
[
1
]
],
"b2a": [
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "sp-eow-split",
"group": "sentencepiece",
"api": "get_alignments",
"normalizer": "default",
"a": [
"going"
],
"b": [
"go",
"ing</w>"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "bpe-basic",
"group": "byte-bpe",
"api": "get_alignments",
"normalizer": "default",
"a": [
"dog",
"walks"
],
"b": [
"dog",
"Ġwalks"
],
"expected": {
"a2b": [
[
0
],
[
1
]
],
"b2a": [
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "bpe-standalone-g",
"group": "byte-bpe",
"api": "get_alignments",
"normalizer": "default",
"a": [
"g",
"ag"
],
"b": [
"Ġ",
"gag"
],
"expected": {
"a2b": [
[
1
],
[
1
]
],
"b2a": [
[],
[
0,
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "prior behavior glues token g to marker G-dot via NFKD residue"
},
{
"id": "bpe-newline",
"group": "byte-bpe",
"api": "get_alignments",
"normalizer": "default",
"a": [
"hello",
"world"
],
"b": [
"hello",
"ĊĊ",
"world"
],
"expected": {
"a2b": [
[
0
],
[
2
]
],
"b2a": [
[
0
],
[],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "bpe-indent",
"group": "byte-bpe",
"api": "get_alignments",
"normalizer": "default",
"a": [
"return",
"x"
],
"b": [
"return",
"ĠĠĠ",
"Ġx"
],
"expected": {
"a2b": [
[
0
],
[
2
]
],
"b2a": [
[
0
],
[],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "bpe-maltese-word",
"group": "byte-bpe",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Ġgant"
],
"b": [
"Ġgant"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "Maltese G-dot dropped on both sides, aligns via the rest"
},
{
"id": "bpe-maltese-lone",
"group": "byte-bpe",
"api": "get_charmap",
"normalizer": "default",
"a": "Ġgant",
"b": "Ġgant",
"expected": {
"a2b": [
[],
[
1
],
[
2
],
[
3
],
[
4
]
],
"b2a": [
[],
[
1
],
[
2
],
[
3
],
[
4
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "char-level, the dropped Maltese initial letter itself gets no counterpart"
},
{
"id": "bpe-endoftext",
"group": "byte-bpe",
"api": "get_alignments",
"normalizer": "default",
"a": [
"done"
],
"b": [
"done",
"<|endoftext|>"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "bpe-nonascii",
"group": "byte-bpe",
"api": "get_alignments",
"normalizer": "default",
"a": [
"café"
],
"b": [
"caf",
"é"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "byte-mapped non-ASCII needs a caller normalizer, default aligns the ASCII prefix only"
},
{
"id": "nfd-eacute-token",
"group": "unicode-nfc-nfd",
"api": "get_alignments",
"normalizer": "default",
"a": [
"café"
],
"b": [
"café"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nfd-vietnamese",
"group": "unicode-nfc-nfd",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Việt",
"Nam"
],
"b": [
"Việt",
"Nam"
],
"expected": {
"a2b": [
[
0
],
[
1
]
],
"b2a": [
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nfd-multi-mark",
"group": "unicode-nfc-nfd",
"api": "get_alignments",
"normalizer": "default",
"a": [
"ệ"
],
"b": [
"ệ"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nfd-nonc-order",
"group": "unicode-nfc-nfd",
"api": "get_alignments",
"normalizer": "default",
"a": [
"ệ"
],
"b": [
"ệ"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "un-normalized mark order, base still pairs, one mark pairs by first-fit"
},
{
"id": "nfd-hangul",
"group": "unicode-nfc-nfd",
"api": "get_alignments",
"normalizer": "default",
"a": [
"한국"
],
"b": [
"한국"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-hangul-jamo",
"group": "unicode-nfc-nfd",
"api": "get_charmap",
"normalizer": "default",
"a": "한",
"b": "한",
"expected": {
"a2b": [
[
0,
1,
2
]
],
"b2a": [
[
0
],
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nfd-greek",
"group": "unicode-nfc-nfd",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Ελλάδα"
],
"b": [
"Ελλάδα"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-angstrom",
"group": "unicode-nfc-nfd",
"api": "get_charmap",
"normalizer": "default",
"a": "Å",
"b": "å",
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-viet-nfd",
"group": "unicode-nfc-nfd",
"api": "get_charmap",
"normalizer": "default",
"a": "Việt",
"b": "Việt",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2,
3,
4
],
[
5
]
],
"b2a": [
[
0
],
[
1
],
[
2
],
[
2
],
[
2
],
[
3
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-ligature",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"file"
],
"b": [
"file"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-fullwidth-token",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Ｈｅｌｌｏ"
],
"b": [
"hello"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-circled",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"①",
"②"
],
"b": [
"1",
"2"
],
"expected": {
"a2b": [
[
0
],
[
1
]
],
"b2a": [
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-half-frac",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"½"
],
"b": [
"1",
"/",
"2"
],
"expected": {
"a2b": [
[
0,
2
]
],
"b2a": [
[
0
],
[],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "fraction slash U+2044 has no compat mapping to solidus, partial by design"
},
{
"id": "compat-roman",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Ⅻ"
],
"b": [
"XII"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-halfwidth-kana",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"ｶﾞｷﾞ"
],
"b": [
"ガギ"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-super",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"x²"
],
"b": [
"x",
"2"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-numero",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"№5"
],
"b": [
"No",
"5"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-cyr-modifier",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"𞀰"
],
"b": [
"а"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-kelvin-ohm",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"KΩ"
],
"b": [
"k",
"ω"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "compat-square-m2",
"group": "unicode-compat",
"api": "get_alignments",
"normalizer": "default",
"a": [
"㎡"
],
"b": [
"m",
"2"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "case-basic",
"group": "case",
"api": "get_alignments",
"normalizer": "default",
"a": [
"HELLO",
"World"
],
"b": [
"hello",
"world"
],
"expected": {
"a2b": [
[
0
],
[
1
]
],
"b2a": [
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "case-sigma-token",
"group": "case",
"api": "get_alignments",
"normalizer": "default",
"a": [
"ΟΔΥΣΣΕΥΣ"
],
"b": [
"οδυσσευς"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "case-sigma-charmap",
"group": "case",
"api": "get_charmap",
"normalizer": "default",
"a": "Σ",
"b": "ς",
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "prior behavior lowercases isolated Sigma to sigma but never folds final sigma, contract folds both"
},
{
"id": "case-turkish",
"group": "case",
"api": "get_alignments",
"normalizer": "default",
"a": [
"İstanbul"
],
"b": [
"istanbul"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "case-eszett",
"group": "case",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Straße"
],
"b": [
"strasse"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "no eszett to ss fold, alignment still lands via the other letters"
},
{
"id": "case-digraph",
"group": "case",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Džungla"
],
"b": [
"džungla"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "case-mixed",
"group": "case",
"api": "get_alignments",
"normalizer": "default",
"a": [
"iPhone"
],
"b": [
"i",
"##phone"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cjk-zh-chars",
"group": "cjk",
"api": "get_alignments",
"normalizer": "default",
"a": [
"北京",
"大学"
],
"b": [
"北",
"京",
"大",
"学"
],
"expected": {
"a2b": [
[
0,
1
],
[
2,
3
]
],
"b2a": [
[
0
],
[
0
],
[
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cjk-zh-resegment",
"group": "cjk",
"api": "get_alignments",
"normalizer": "default",
"a": [
"研究",
"生命"
],
"b": [
"研究生",
"命"
],
"expected": {
"a2b": [
[
0
],
[
0,
1
]
],
"b2a": [
[
0,
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cjk-ja-mixed",
"group": "cjk",
"api": "get_alignments",
"normalizer": "default",
"a": [
"東京タワー",
"へ"
],
"b": [
"東京",
"タワー",
"へ"
],
"expected": {
"a2b": [
[
0,
1
],
[
2
]
],
"b2a": [
[
0
],
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cjk-fullwidth-punct",
"group": "cjk",
"api": "get_alignments",
"normalizer": "default",
"a": [
"こんにちは",
"。"
],
"b": [
"こんにちは。"
],
"expected": {
"a2b": [
[
0
],
[
0
]
],
"b2a": [
[
0,
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cjk-wordpiece",
"group": "cjk",
"api": "get_alignments",
"normalizer": "default",
"a": [
"北京"
],
"b": [
"北",
"##京"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cjk-katakana-choon",
"group": "cjk",
"api": "get_alignments",
"normalizer": "default",
"a": [
"コーヒー"
],
"b": [
"コ",
"ー",
"ヒ",
"ー"
],
"expected": {
"a2b": [
[
0,
1,
2,
3
]
],
"b2a": [
[
0
],
[
0
],
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cjk-ideo-space",
"group": "cjk",
"api": "get_alignments",
"normalizer": "default",
"a": [
"日本",
"語"
],
"b": [
"日本 語"
],
"expected": {
"a2b": [
[
0
],
[
0
]
],
"b2a": [
[
0,
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "emoji-zwj-family",
"group": "emoji",
"api": "get_alignments",
"normalizer": "default",
"a": [
"👨👩👧"
],
"b": [
"👨",
"",
"👩",
"",
"👧"
],
"expected": {
"a2b": [
[
0,
1,
2,
3,
4
]
],
"b2a": [
[
0
],
[
0
],
[
0
],
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "emoji-zwj-identity",
"group": "emoji",
"api": "get_alignments",
"normalizer": "default",
"a": [
"👩💻",
"rocks"
],
"b": [
"👩💻",
"rocks"
],
"expected": {
"a2b": [
[
0
],
[
1
]
],
"b2a": [
[
0
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "emoji-skintone",
"group": "emoji",
"api": "get_alignments",
"normalizer": "default",
"a": [
"👍🏽"
],
"b": [
"👍",
"🏽"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "emoji-vs16",
"group": "emoji",
"api": "get_alignments",
"normalizer": "default",
"a": [
"❤️"
],
"b": [
"❤"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "emoji-flag-tags",
"group": "emoji",
"api": "get_alignments",
"normalizer": "default",
"a": [
"🏴"
],
"b": [
"🏴"
],
"expected": {
"a2b": [
[
0
]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-empty-both",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [],
"b": [],
"expected": {
"a2b": [],
"b2a": []
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-empty-a",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [],
"b": [
"a",
"b"
],
"expected": {
"a2b": [],
"b2a": [
[],
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-empty-b",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"a",
"b"
],
"b": [],
"expected": {
"a2b": [
[],
[]
],
"b2a": []
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-empty-strings",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"a",
"",
"b"
],
"b": [
"ab"
],
"expected": {
"a2b": [
[
0
],
[],
[
0
]
],
"b2a": [
[
0,
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-disjoint",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"cat"
],
"b": [
"dog"
],
"expected": {
"a2b": [
[]
],
"b2a": [
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-whitespace-token",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"a",
" ",
"b"
],
"b": [
"a",
"b"
],
"expected": {
"a2b": [
[
0
],
[],
[
1
]
],
"b2a": [
[
0
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-whitespace-match",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"a",
" ",
"b"
],
"b": [
"a",
" ",
"b"
],
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-marker-run",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"x",
"y"
],
"b": [
"x",
"▁▁",
"y"
],
"expected": {
"a2b": [
[
0
],
[
2
]
],
"b2a": [
[
0
],
[],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-dropped-punct",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"Hello",
",",
"world"
],
"b": [
"hello",
"world"
],
"expected": {
"a2b": [
[
0
],
[],
[
1
]
],
"b2a": [
[
0
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-numbers",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"3.14"
],
"b": [
"3",
"14"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "nc-total-mismatch",
"group": "no-counterpart",
"api": "get_alignments",
"normalizer": "default",
"a": [
"aaa",
"bbb",
"ccc"
],
"b": [
"zzz"
],
"expected": {
"a2b": [
[],
[],
[]
],
"b2a": [
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "po-merge",
"group": "partial-overlap",
"api": "get_alignments",
"normalizer": "default",
"a": [
"to",
"get",
"her"
],
"b": [
"together"
],
"expected": {
"a2b": [
[
0
],
[
0
],
[
0
]
],
"b2a": [
[
0,
1,
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "po-shift",
"group": "partial-overlap",
"api": "get_alignments",
"normalizer": "default",
"a": [
"ab",
"cd"
],
"b": [
"a",
"bc",
"d"
],
"expected": {
"a2b": [
[
0,
1
],
[
1,
2
]
],
"b2a": [
[
0
],
[
0,
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "po-overlap-chain",
"group": "partial-overlap",
"api": "get_alignments",
"normalizer": "default",
"a": [
"abc",
"def"
],
"b": [
"ab",
"cde",
"f"
],
"expected": {
"a2b": [
[
0,
1
],
[
1,
2
]
],
"b2a": [
[
0
],
[
0,
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "po-one-to-many",
"group": "partial-overlap",
"api": "get_alignments",
"normalizer": "default",
"a": [
"internationalization"
],
"b": [
"inter",
"national",
"ization"
],
"expected": {
"a2b": [
[
0,
1,
2
]
],
"b2a": [
[
0
],
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "po-uneven",
"group": "partial-overlap",
"api": "get_alignments",
"normalizer": "default",
"a": [
"a",
"bcdef"
],
"b": [
"abcde",
"f"
],
"expected": {
"a2b": [
[
0
],
[
0,
1
]
],
"b2a": [
[
0,
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "po-spacefree-merge",
"group": "partial-overlap",
"api": "get_alignments",
"normalizer": "default",
"a": [
"New",
"York",
"City"
],
"b": [
"NewYork",
"City"
],
"expected": {
"a2b": [
[
0
],
[
0
],
[
1
]
],
"b2a": [
[
0,
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "amb-interior-repeat",
"group": "ambiguous",
"api": "get_alignments",
"normalizer": "default",
"a": [
"1",
"a",
"b",
"a",
"2"
],
"b": [
"3",
"a",
"4"
],
"expected": {
"a2b": [
[],
[],
[],
[
1
],
[]
],
"b2a": [
[],
[
3
],
[]
]
},
"ambiguous": true,
"ambiguity_check": "exhaustive",
"alternatives": [
{
"a2b": [
[],
[
1
],
[],
[],
[]
],
"b2a": [
[],
[
1
],
[]
]
}
],
"note": "two maximum matchings project differently, both accepted"
},
{
"id": "amb-charmap-repeat",
"group": "ambiguous",
"api": "get_charmap",
"normalizer": "default",
"a": "1aba2",
"b": "3a4",
"expected": {
"a2b": [
[],
[],
[],
[
1
],
[]
],
"b2a": [
[],
[
3
],
[]
]
},
"ambiguous": true,
"ambiguity_check": "exhaustive",
"alternatives": [
{
"a2b": [
[],
[
1
],
[],
[],
[]
],
"b2a": [
[],
[
1
],
[]
]
}
],
"note": "char-level twin of amb-interior-repeat"
},
{
"id": "amb-resolved-boundary",
"group": "ambiguous",
"api": "get_alignments",
"normalizer": "default",
"a": [
"x",
"a",
"b",
"a",
"y"
],
"b": [
"x",
"a",
"y"
],
"expected": {
"a2b": [
[
0
],
[
1
],
[],
[],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
4
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "the repeat sits against the matched prefix, boundary rule leaves one projection"
},
{
"id": "amb-resolved-prefix",
"group": "ambiguous",
"api": "get_alignments",
"normalizer": "default",
"a": [
"a",
"a"
],
"b": [
"a"
],
"expected": {
"a2b": [
[
0
],
[]
],
"b2a": [
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "boundary rule pins the prefix pair, not ambiguous under the contract"
},
{
"id": "noise-unknown-marker",
"group": "ambiguous",
"api": "get_alignments",
"normalizer": "default",
"a": [
"e",
"g"
],
"b": [
"e",
"@@",
"g"
],
"expected": {
"a2b": [
[
0
],
[
2
]
],
"b2a": [
[
0
],
[],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "unknown marker under the default, resolved by boundary rule"
},
{
"id": "cm-space-identity",
"group": "charmap",
"api": "get_charmap",
"normalizer": "default",
"a": "a b",
"b": "a b",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-eacute-strip",
"group": "charmap",
"api": "get_charmap",
"normalizer": "default",
"a": "café",
"b": "cafe",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
],
[
3
]
],
"b2a": [
[
0
],
[
1
],
[
2
],
[
3
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-leading-space",
"group": "charmap",
"api": "get_charmap",
"normalizer": "default",
"a": " a",
"b": "a",
"expected": {
"a2b": [
[],
[
0
]
],
"b2a": [
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-emoji-prefix",
"group": "charmap",
"api": "get_charmap",
"normalizer": "default",
"a": "🍕x",
"b": "x",
"expected": {
"a2b": [
[],
[
0
]
],
"b2a": [
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-newline-vs-space",
"group": "charmap",
"api": "get_charmap",
"normalizer": "default",
"a": "a\nb",
"b": "a b",
"expected": {
"a2b": [
[
0
],
[],
[
2
]
],
"b2a": [
[
0
],
[],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-nbsp",
"group": "charmap",
"api": "get_charmap",
"normalizer": "default",
"a": "a b",
"b": "a b",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
]
],
"b2a": [
[
0
],
[
1
],
[
2
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "cm-long-identity",
"group": "charmap",
"api": "get_charmap",
"normalizer": "default",
"a": "The quick brown fox jumps over the lazy dog while 42 ships sailed to 東京.",
"b": "The quick brown fox jumps over the lazy dog while 42 ships sailed to 東京.",
"expected": {
"a2b": [
[
0
],
[
1
],
[
2
],
[
3
],
[
4
],
[
5
],
[
6
],
[
7
],
[
8
],
[
9
],
[
10
],
[
11
],
[
12
],
[
13
],
[
14
],
[
15
],
[
16
],
[
17
],
[
18
],
[
19
],
[
20
],
[
21
],
[
22
],
[
23
],
[
24
],
[
25
],
[
26
],
[
27
],
[
28
],
[
29
],
[
30
],
[
31
],
[
32
],
[
33
],
[
34
],
[
35
],
[
36
],
[
37
],
[
38
],
[
39
],
[
40
],
[
41
],
[
42
],
[
43
],
[
44
],
[
45
],
[
46
],
[
47
],
[
48
],
[
49
],
[
50
],
[
51
],
[
52
],
[
53
],
[
54
],
[
55
],
[
56
],
[
57
],
[
58
],
[
59
],
[
60
],
[
61
],
[
62
],
[
63
],
[
64
],
[
65
],
[
66
],
[
67
],
[
68
],
[
69
],
[
70
],
[
71
]
],
"b2a": [
[
0
],
[
1
],
[
2
],
[
3
],
[
4
],
[
5
],
[
6
],
[
7
],
[
8
],
[
9
],
[
10
],
[
11
],
[
12
],
[
13
],
[
14
],
[
15
],
[
16
],
[
17
],
[
18
],
[
19
],
[
20
],
[
21
],
[
22
],
[
23
],
[
24
],
[
25
],
[
26
],
[
27
],
[
28
],
[
29
],
[
30
],
[
31
],
[
32
],
[
33
],
[
34
],
[
35
],
[
36
],
[
37
],
[
38
],
[
39
],
[
40
],
[
41
],
[
42
],
[
43
],
[
44
],
[
45
],
[
46
],
[
47
],
[
48
],
[
49
],
[
50
],
[
51
],
[
52
],
[
53
],
[
54
],
[
55
],
[
56
],
[
57
],
[
58
],
[
59
],
[
60
],
[
61
],
[
62
],
[
63
],
[
64
],
[
65
],
[
66
],
[
67
],
[
68
],
[
69
],
[
70
],
[
71
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "hook-exact-nomatch",
"group": "hook",
"api": "get_alignments",
"normalizer": "exact",
"a": [
"AB"
],
"b": [
"ab"
],
"expected": {
"a2b": [
[]
],
"b2a": [
[]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "identity normalizer, case difference no longer matches"
},
{
"id": "hook-exact-match",
"group": "hook",
"api": "get_alignments",
"normalizer": "exact",
"a": [
"ab",
"cd"
],
"b": [
"abc",
"d"
],
"expected": {
"a2b": [
[
0
],
[
0,
1
]
],
"b2a": [
[
0,
1
],
[
1
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive"
},
{
"id": "hook-custom-atat",
"group": "hook",
"api": "get_alignments",
"normalizer": "strip_atat",
"a": [
"hashtag"
],
"b": [
"hash@@",
"tag"
],
"expected": {
"a2b": [
[
0,
1
]
],
"b2a": [
[
0
],
[
0
]
]
},
"ambiguous": false,
"ambiguity_check": "exhaustive",
"note": "example custom normalizer stripping subword-nmt continuation"
}
]
}