token-align 0.1.0

Maps overlapping tokens and character positions between tokenized texts
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

mod common;

#[test]
fn all_119_vectors_match() {
    let cases = common::cases();
    assert_eq!(cases.len(), 119);
    for case in &cases {
        common::assert_case(case);
    }
}

#[test]
fn defect_d1_standalone_marker_does_not_capture_text() {
    common::assert_case(&common::find("bpe-standalone-g"));
}

#[test]
fn defect_d2_control_tokens_never_align_by_shared_letters() {
    for id in ["wp-mask", "wp-cls-steal-l", "wp-sep-steal-s"] {
        common::assert_case(&common::find(id));
    }
}

#[test]
fn defect_d3_unknown_token_has_no_counterpart() {
    common::assert_case(&common::find("unk-luck"));
}

#[test]
fn defect_d4_final_sigma_matches_sigma() {
    common::assert_case(&common::find("case-sigma-charmap"));
}

#[test]
fn defect_d5_wordpiece_collision_is_explicit() {
    common::assert_case(&common::find("wp-collision"));
}

#[test]
fn defect_d6_orphan_marks_do_not_block_base_matches() {
    for id in ["wp-uncased-accent", "cm-eacute-strip"] {
        common::assert_case(&common::find(id));
    }
}

#[test]
fn api_parity_group_has_18_exact_cases() {
    let cases = common::cases();
    let parity: Vec<_> = cases
        .iter()
        .filter(|case| case.group == "incumbent-parity")
        .collect();
    assert_eq!(parity.len(), 18);
    for case in parity {
        assert!(case.alternatives.is_empty());
        common::assert_case(case);
    }
}