symproj 0.1.4

Codebook-based token-to-vector projection
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

//! Compare symproj's `l2_normalize_in_place` with innr's `normalize`.
//!
//! Both should produce identical unit vectors. Without symproj's `simd` feature,
//! this compares symproj's scalar implementation against innr. With the feature,
//! `l2_normalize_in_place` delegates to innr and this checks the delegated path.

use innr::{norm, normalize as innr_normalize};
use symproj::{l2_normalize_in_place, Codebook};

fn main() {
    let dim = 64;
    let vocab_size = 8;

    // Deterministic pseudo-random matrix (LCG).
    let mut matrix = Vec::with_capacity(vocab_size * dim);
    let mut state: u64 = 42;
    for _ in 0..vocab_size * dim {
        state = state
            .wrapping_mul(6_364_136_223_846_793_005)
            .wrapping_add(1);
        let val = ((state >> 33) as f32) / (u32::MAX as f32) - 0.5;
        matrix.push(val);
    }

    let codebook = Codebook::new(matrix, dim).expect("valid codebook");
    println!(
        "Codebook: vocab_size={}, dim={dim}\n",
        codebook.vocab_size()
    );
    println!(
        "symproj simd feature: {}\n",
        if cfg!(feature = "simd") {
            "enabled"
        } else {
            "disabled"
        }
    );

    // Encode a few token-id sequences and normalize both ways.
    let sequences: &[&[u32]] = &[&[0, 1, 2], &[3, 4, 5, 6, 7], &[0, 7], &[2]];

    for ids in sequences {
        let raw = codebook.encode_ids(ids);

        // symproj path
        let mut v_symproj = raw.clone();
        l2_normalize_in_place(&mut v_symproj);

        // innr path
        let mut v_innr = raw.clone();
        innr_normalize(&mut v_innr);

        // Both should be unit vectors.
        let norm_symproj = norm(&v_symproj);
        let norm_innr = norm(&v_innr);

        // Element-wise max absolute difference.
        let max_diff: f32 = v_symproj
            .iter()
            .zip(v_innr.iter())
            .map(|(a, b)| (a - b).abs())
            .fold(0.0f32, f32::max);

        println!("ids={ids:?}");
        println!("  norm (symproj): {norm_symproj:.8}");
        println!("  norm (innr):    {norm_innr:.8}");
        println!("  max element diff: {max_diff:.2e}");
        assert!(
            max_diff < 1e-6,
            "normalization methods diverged: max_diff={max_diff}"
        );
        println!("  PASSED\n");
    }

    println!("All vectors match between symproj and innr normalization.");
}