katgpt_types/
kv_cache.rs

1//! Quantized KV cache trait — shared extension point for all backends.
2//!
3//! Originally lived in `katgpt-rs/src/types.rs` (Plan 123 / Issue 015 Phase 1).
4//! Promoted to `katgpt-types` so that every KV backend crate
5//! (`katgpt-kv`, sibling engine crates, future downstream) can implement
6//! against a stable leaf-crate interface without depending on the root
7//! `katgpt-rs` crate.
8//!
9//! The `compact_into` extension (which historically gated on
10//! `crate::still_kv::CompactionStrategy`) is intentionally NOT in this
11//! trait. It lives in `katgpt-kv` behind the `still_kv` feature as the
12//! `CompactableKVCache` extension trait, so this leaf crate stays free
13//! of any KV-storage-concrete type coupling.
14
15/// Shared interface for quantized KV caches.
16///
17/// Enables `transformer::forward_quantized` to work with any compression
18/// backend (TurboQuant, SpectralQuant, OscKV, ShardKV, KVarN, or future
19/// methods). Backends implement this trait; the inference loop stays
20/// backend-agnostic.
21pub trait QuantizedKVCache {
22    /// Quantize and store a key vector at given layer and position.
23    fn store_key(&mut self, layer: usize, pos: usize, key: &[f32]);
24    /// Quantize and store a value vector at given layer and position.
25    fn store_value(&mut self, layer: usize, pos: usize, value: &[f32]);
26    /// Dequantize a key into a pre-allocated buffer (zero-alloc hot path).
27    fn dequantize_key_into(&mut self, layer: usize, pos: usize, out: &mut [f32]);
28    /// Dequantize a value into a pre-allocated buffer (zero-alloc hot path).
29    fn dequantize_value_into(&mut self, layer: usize, pos: usize, out: &mut [f32]);
30    /// Reset cache for a new sequence.
31    fn reset(&mut self);
32    /// Current write position.
33    fn pos(&self) -> usize;
34    /// Set the current write position.
35    fn set_pos(&mut self, pos: usize);
36}
katgpt_types/kv_cache.rs

katgpt_types/
kv_cache.rs
