llmp 0.1.0

Measures how well a local LLM setup uses its model: a recording proxy and a prompt-reuse analysis
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

use serde_json::Value;

const OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
const PRIME: u64 = 0x0000_0100_0000_01b3;

pub(crate) fn fnv1a64(bytes: &[u8]) -> u64 {
    bytes.iter().fold(OFFSET_BASIS, |hash, &byte| {
        (hash ^ u64::from(byte)).wrapping_mul(PRIME)
    })
}

pub(crate) fn hex(bytes: &[u8]) -> String {
    format!("{:016x}", fnv1a64(bytes))
}

pub(crate) fn hex_value(value: &Value) -> String {
    hex(canonical_json(value).as_bytes())
}

pub(crate) fn canonical_json(value: &Value) -> String {
    let mut out = String::new();
    write_canonical(value, &mut out);
    out
}

fn write_canonical(value: &Value, out: &mut String) {
    match value {
        Value::Array(items) => {
            out.push('[');
            for (i, item) in items.iter().enumerate() {
                if i > 0 {
                    out.push(',');
                }
                write_canonical(item, out);
            }
            out.push(']');
        }
        Value::Object(map) => {
            let mut keys: Vec<&String> = map.keys().collect();
            keys.sort();
            out.push('{');
            for (i, key) in keys.into_iter().enumerate() {
                if i > 0 {
                    out.push(',');
                }
                out.push_str(&Value::String(key.clone()).to_string());
                out.push(':');
                if let Some(item) = map.get(key) {
                    write_canonical(item, out);
                }
            }
            out.push('}');
        }
        scalar => out.push_str(&scalar.to_string()),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn fnv1a64_reference_vectors() {
        assert_eq!(fnv1a64(b""), 0xcbf2_9ce4_8422_2325);
        assert_eq!(fnv1a64(b"a"), 0xaf63_dc4c_8601_ec8c);
        assert_eq!(fnv1a64(b"foobar"), 0x8594_4171_f739_67e8);
    }

    #[test]
    fn hex_is_sixteen_lowercase_digits() {
        assert_eq!(hex(b"a"), "af63dc4c8601ec8c");
        assert_eq!(hex(b""), "cbf29ce484222325");
    }

    #[test]
    fn canonical_json_sorts_keys_at_every_depth() {
        let a = json!({"b": 1, "a": {"y": [1, {"d": null, "c": "x"}], "x": true}});
        assert_eq!(
            canonical_json(&a),
            r#"{"a":{"x":true,"y":[1,{"c":"x","d":null}]},"b":1}"#
        );
    }

    #[test]
    fn canonical_json_escapes_strings() {
        let v = json!({"k\"ey": "line\nbreak \u{e9}"});
        assert_eq!(canonical_json(&v), "{\"k\\\"ey\":\"line\\nbreak \u{e9}\"}");
    }

    #[test]
    fn key_order_does_not_change_the_hash() {
        let a: Value = serde_json::from_str(r#"{"role":"user","content":"hi"}"#).unwrap();
        let b: Value = serde_json::from_str(r#"{"content":"hi","role":"user"}"#).unwrap();
        assert_eq!(hex_value(&a), hex_value(&b));
    }
}