Struct KvParams 

pub struct KvParams {
    pub kind: Kind,
    pub heads: usize,
    pub hd: usize,
    pub theta: f64,
    pub pre_rope: bool,
    pub step: f64,
    pub sink: usize,
    pub sink_div: u32,
    pub block: usize,
    pub ar: bool,
    pub gpu: bool,
    pub lane_ch: usize,
}

Encoder settings.

Fields

kind: Kind

K or V.

heads: usize

Number of KV heads.

hd: usize

Head dimension (even).

theta: f64

RoPE base (HF-Llama half-split convention).

pre_rope: bool

Code K before RoPE (undo, code, redo). Ignored for V.

step: f64

Quantizer step in data units (max abs error is step / 2).

sink: usize

Number of leading absolute positions treated as attention sinks.

sink_div: u32

Step divisor for sink positions.

block: usize

Tokens per independent block.

ar: bool

Fit a per-channel AR(1) predictor (else predict the channel mean).

gpu: bool

GPU layout: integer reconstruction, one substream per (block, head).

lane_ch: usize

GPU layout: channels per substream (even, divides hd; 0 = hd). Each block then has heads × hd / lane_ch substreams; lane g of a head holds the RoPE pairs g·lane_ch/2 .. (channels c and c + hd/2 together) so the decoder can rotate on write.

Implementations

impl KvParams

pub fn new(kind: Kind, heads: usize, hd: usize, step: f64) -> Self

Defaults for a cache with heads × hd per token (SmolLM2 RoPE base).

Examples found in repository

examples/kvc.rs (line 54)

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 4 {
        usage()
    }
    match args[1].as_str() {
        "encode" => {
            let x = read_f32(&args[2]);
            let shape: Vec<usize> = flag(&args, "--shape")
                .unwrap_or_else(|| usage())
                .split('x')
                .map(|s| s.parse().unwrap())
                .collect();
            let (h, n, d) = (shape[0], shape[1], shape[2]);
            let kind = match flag(&args, "--kind").as_deref() {
                Some("k") => Kind::K,
                Some("v") => Kind::V,
                _ => usage(),
            };
            let mut p = KvParams::new(kind, h, d, 1.0);
            p.pre_rope = !args.iter().any(|a| a == "--post");
            p.ar = !args.iter().any(|a| a == "--noar");
            p.gpu = args.iter().any(|a| a == "--gpu");
            if let Some(v) = flag(&args, "--sink") {
                p.sink = v.parse().unwrap();
            }
            if let Some(v) = flag(&args, "--sinkdiv") {
                p.sink_div = v.parse().unwrap();
            }
            if let Some(v) = flag(&args, "--block") {
                p.block = v.parse().unwrap();
            }
            if let Some(v) = flag(&args, "--theta") {
                p.theta = v.parse().unwrap();
            }
            let t = Instant::now();
            p.step = if let Some(r) = flag(&args, "--rmse") {
                r.parse::<f64>().unwrap() * 12f64.sqrt()
            } else if let Some(e) = flag(&args, "--maxabs") {
                // pre-RoPE pair rotation can move a per-element error by sqrt(2)
                let e: f64 = e.parse().unwrap();
                if kind == Kind::K && p.pre_rope {
                    e * std::f64::consts::SQRT_2
                } else {
                    2.0 * e
                }
            } else if let Some(b) = flag(&args, "--bpv") {
                step_for_bpv(p.clone(), &x, n, b.parse().unwrap())
            } else {
                usage()
            };
            let pre: usize = flag(&args, "--prefill")
                .map(|v| v.parse().unwrap())
                .unwrap_or(n);
            let slice = |t0: usize, t1: usize| -> Vec<f32> {
                let mut v = Vec::with_capacity(h * (t1 - t0) * d);
                for hh in 0..h {
                    v.extend_from_slice(&x[(hh * n + t0) * d..(hh * n + t1) * d]);
                }
                v
            };
            let mut e = KvEncoder::new(p.clone(), &slice(0, pre), pre);
            let ta = Instant::now();
            for t in pre..n {
                e.append(&slice(t, t + 1), 1);
            }
            let append_s = ta.elapsed().as_secs_f64();
            let bytes = e.to_bytes();
            let enc = t.elapsed().as_secs_f64();
            std::fs::write(&args[3], &bytes).unwrap();
            println!(
                "{{\"bytes\":{},\"bpv\":{},\"step\":{},\"encode_s\":{enc},\"append_s\":{append_s},\"appended\":{}}}",
                bytes.len(),
                8.0 * bytes.len() as f64 / x.len() as f64,
                p.step,
                n - pre
            );
        }
        "decode" => {
            let bytes = std::fs::read(&args[2]).unwrap();
            let s = KvStream::parse(&bytes).unwrap();
            let (t0, t1) = match flag(&args, "--range") {
                Some(r) => {
                    let (a, b) = r.split_once(':').unwrap();
                    (a.parse().unwrap(), b.parse().unwrap())
                }
                None => (0, s.tokens),
            };
            let reps: usize = flag(&args, "--reps")
                .map(|v| v.parse().unwrap())
                .unwrap_or(1);
            let t = Instant::now();
            let mut y = Vec::new();
            for _ in 0..reps {
                y = s.decode_range(t0, t1);
            }
            let dec = t.elapsed().as_secs_f64() / reps as f64;
            let out: Vec<u8> = y.iter().flat_map(|v| v.to_le_bytes()).collect();
            std::fs::write(&args[3], out).unwrap();
            println!(
                "{{\"decode_s\":{dec},\"ns_per_value\":{}}}",
                1e9 * dec / y.len().max(1) as f64
            );
        }
        _ => usage(),
    }
}

pub fn lane_width(&self) -> usize

Channels per GPU substream.

pub fn lane_groups(&self) -> usize

GPU substreams per (block, head).

pub fn unit(&self) -> f32

GPU-layout unit: every reconstruction is an i32 multiple of it.

Trait Implementations

impl Clone for KvParams

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for KvParams

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.