token-align 0.1.0

Maps overlapping tokens and character positions between tokenized texts
Documentation
use proptest::collection::vec;
use proptest::prelude::*;
use token_align::{get_alignments, get_charmap, Alignment};

fn token() -> impl Strategy<Value = String> {
    prop_oneof![
        any::<String>(),
        Just(String::new()),
        Just("##โ–ฤ ฤŠ</w>".to_owned()),
        Just("\u{0301}\u{200D}\u{FE0F}".to_owned()),
        Just("\u{FDD0}\u{10FFFF}".to_owned()),
        Just("๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ".to_owned()),
    ]
}

proptest! {
    #![proptest_config(ProptestConfig {
        cases: 256,
        failure_persistence: None,
        ..ProptestConfig::default()
    })]

    #[test]
    fn arbitrary_tokens_preserve_alignment_invariants(
        a in vec(token(), 0..20),
        b in vec(token(), 0..20),
    ) {
        let first = get_alignments(&a, &b);
        let second = get_alignments(&a, &b);
        prop_assert_eq!(&first, &second);
        check(&first.0, &first.1, a.len(), b.len())?;
    }

    #[test]
    fn arbitrary_text_has_in_bounds_character_maps(a in any::<String>(), b in any::<String>()) {
        let (a2b, b2a) = get_charmap(&a, &b);
        let a_len = a.chars().count();
        let b_len = b.chars().count();
        check(&a2b, &b2a, a_len, b_len)?;
    }
}

fn check(
    a2b: &Alignment,
    b2a: &Alignment,
    a_len: usize,
    b_len: usize,
) -> Result<(), TestCaseError> {
    prop_assert_eq!(a2b.len(), a_len);
    prop_assert_eq!(b2a.len(), b_len);
    for row in a2b {
        prop_assert!(row.windows(2).all(|pair| pair[0] < pair[1]));
        prop_assert!(row.iter().all(|&index| index < b_len));
    }
    for row in b2a {
        prop_assert!(row.windows(2).all(|pair| pair[0] < pair[1]));
        prop_assert!(row.iter().all(|&index| index < a_len));
    }
    let mut previous_max = None;
    for row in a2b.iter().filter(|row| !row.is_empty()) {
        if let Some(previous) = previous_max {
            prop_assert!(previous <= row[0]);
        }
        previous_max = row.last().copied();
    }
    for (a_index, row) in a2b.iter().enumerate() {
        for &b_index in row {
            prop_assert!(b2a[b_index].contains(&a_index));
        }
    }
    for (b_index, row) in b2a.iter().enumerate() {
        for &a_index in row {
            prop_assert!(a2b[a_index].contains(&b_index));
        }
    }
    Ok(())
}