token-align 0.1.0

use std::fmt::Debug;
use std::hash::Hash;

use token_align::{get_alignments, get_alignments_with, Exact, Fold};

#[test]
fn owned_and_borrowed_token_slices_mix() {
    let owned = vec!["New".to_owned(), "York".to_owned()];
    let borrowed = ["new york"];
    assert_eq!(
        get_alignments(&owned, &borrowed),
        (vec![vec![0], vec![0]], vec![vec![0, 1]])
    );
}

#[test]
fn closure_normalizers_use_the_public_trait_bridge() {
    let left = ["hash@@", "tag"];
    let right = vec!["hashtag".to_owned()];
    let strip_suffix = |token: &str, out: &mut String| {
        out.push_str(token.strip_suffix("@@").unwrap_or(token));
    };
    assert_eq!(
        get_alignments_with(&left, &right, &strip_suffix),
        (vec![vec![0], vec![0]], vec![vec![0, 1]])
    );
}

#[test]
fn fold_and_exact_implement_common_value_traits() {
    fn assert_traits<T>()
    where
        T: Clone + Copy + Debug + Default + Eq + Hash + Ord + PartialEq + PartialOrd,
    {
    }

    assert_traits::<Fold>();
    assert_traits::<Exact>();
}

#[test]
fn fold_docs_cover_lossy_rules_and_custom_decoder() {
    let source = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/lib.rs"))
        .expect("read library source");
    for required in [
        "[CLS]",
        "<|startoftext|>",
        "one trailing `</w>`",
        "Every `▁`, `Ġ`, and",
        "Maltese capital letters",
        "Unicode Character Database 17.0.0",
        "recursive compatibility decomposition",
        "simple\n/// lowercase field",
        "Final sigma `ς` maps to sigma `σ`",
        "struct DecodeByteToken",
    ] {
        assert!(
            source.contains(required),
            "missing rustdoc contract: {required}"
        );
    }
}