token-align 0.1.0

Maps overlapping tokens and character positions between tokenized texts
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

#[cfg(not(debug_assertions))]
mod release {
    use std::hint::black_box;
    use std::time::{Duration, Instant};

    use token_align::get_alignments;

    #[test]
    fn marker_shapes_scale_linearly_and_meet_the_release_ceiling() {
        for shape in [Shape::Identical, Shape::WordPiece, Shape::SentencePiece] {
            assert_growth(shape);
        }
    }

    #[test]
    fn accented_marker_shape_scales_linearly_and_meets_the_release_ceiling() {
        assert_growth(Shape::SentencePieceAccent);
    }

    fn assert_growth(shape: Shape) {
        let small = inputs(10_000, shape);
        let large = inputs(100_000, shape);
        let small_time = measure(&small, 5);
        let large_time = measure(&large, 3);
        let ratio = large_time.as_secs_f64() / small_time.as_secs_f64();
        eprintln!("{shape:?}: 10k {small_time:?}, 100k {large_time:?}, ratio {ratio:.2}");
        assert!(
            ratio <= 15.0,
            "{shape:?} growth ratio {ratio:.2} exceeds 15: {small_time:?} to {large_time:?}"
        );
        assert!(
            large_time <= Duration::from_millis(250),
            "{shape:?} took {large_time:?} for 100k source tokens"
        );
    }

    fn measure(input: &(Vec<String>, Vec<String>), repetitions: usize) -> Duration {
        let mut best = Duration::MAX;
        for _ in 0..repetitions {
            let start = Instant::now();
            let result = get_alignments(black_box(&input.0), black_box(&input.1));
            black_box(result);
            best = best.min(start.elapsed());
        }
        best
    }

    #[derive(Clone, Copy, Debug)]
    enum Shape {
        Identical,
        WordPiece,
        SentencePiece,
        SentencePieceAccent,
    }

    fn inputs(count: usize, shape: Shape) -> (Vec<String>, Vec<String>) {
        let left: Vec<String> = match shape {
            Shape::SentencePieceAccent => (0..count)
                .map(|index| format!("café{}", index % 10))
                .collect(),
            _ => (0..count)
                .map(|index| format!("word{}", index % 10))
                .collect(),
        };
        let right = match shape {
            Shape::Identical => left.clone(),
            Shape::WordPiece => (0..count)
                .flat_map(|index| ["wo".to_owned(), format!("##rd{}", index % 10)])
                .collect(),
            Shape::SentencePiece => (0..count)
                .map(|index| format!("▁word{}", index % 10))
                .collect(),
            Shape::SentencePieceAccent => (0..count)
                .map(|index| format!("▁cafe{}", index % 10))
                .collect(),
        };
        (left, right)
    }
}