token-align 0.1.0

Maps overlapping tokens and character positions between tokenized texts
Documentation
use std::alloc::{GlobalAlloc, Layout, System};
use std::sync::atomic::{AtomicUsize, Ordering};

use token_align::get_charmap;

struct CountingAllocator;

static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0);

unsafe impl GlobalAlloc for CountingAllocator {
    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
        ALLOCATIONS.fetch_add(1, Ordering::Relaxed);
        unsafe { System.alloc(layout) }
    }

    unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) {
        unsafe { System.dealloc(pointer, layout) }
    }

    unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
        ALLOCATIONS.fetch_add(1, Ordering::Relaxed);
        unsafe { System.alloc_zeroed(layout) }
    }

    unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, size: usize) -> *mut u8 {
        ALLOCATIONS.fetch_add(1, Ordering::Relaxed);
        unsafe { System.realloc(pointer, layout, size) }
    }
}

#[global_allocator]
static GLOBAL: CountingAllocator = CountingAllocator;

#[test]
fn charmap_normalization_does_not_allocate_per_character() {
    let small = "\u{0301}".repeat(32);
    let large = "\u{0301}".repeat(4_096);

    let small_allocations = measured_allocations(|| get_charmap(&small, ""));
    let large_allocations = measured_allocations(|| get_charmap(&large, ""));

    assert!(
        large_allocations <= small_allocations + 2,
        "allocation count grew with characters: {small_allocations} to {large_allocations}"
    );
    assert!(
        large_allocations <= 12,
        "character normalization used {large_allocations} allocations"
    );
}

fn measured_allocations<T>(operation: impl FnOnce() -> T) -> usize {
    ALLOCATIONS.store(0, Ordering::SeqCst);
    let result = operation();
    let count = ALLOCATIONS.load(Ordering::SeqCst);
    drop(result);
    count
}