1use crate::cache::KvCache;
55use crate::kv_swa::{BlockLayout, BlockLayoutError};
56
57pub const BLOCK_FORMAT_VERSION: u32 = 2;
61
62pub const READABLE_FORMAT_VERSIONS: &[u32] = &[2];
72
73#[derive(Clone, Copy, Debug, PartialEq, Eq)]
80pub enum KvDtype {
81 F32,
82}
83
84impl KvDtype {
85 pub fn as_str(self) -> &'static str {
86 match self {
87 KvDtype::F32 => "f32",
88 }
89 }
90}
91
92#[derive(Clone, Debug, PartialEq, Eq)]
96pub struct CacheSignature {
97 pub format_version: u32,
98 pub model: String,
102 pub n_layers: usize,
103 pub n_kv_heads: usize,
104 pub head_dim: usize,
105 pub dtype: KvDtype,
106 pub tokens: usize,
110 pub layout: BlockLayout,
113}
114
115impl CacheSignature {
116 pub fn from_payload(
127 model: &str,
128 layout: BlockLayout,
129 layers: &[KvCache],
130 ) -> Result<Self, SignatureError> {
131 let first = layers.first().ok_or(SignatureError::EmptyPayload)?;
132 let n_kv_heads = first.n_kv_heads;
133 let head_dim = first.head_dim;
134 if n_kv_heads == 0 || head_dim == 0 {
135 return Err(SignatureError::DegenerateLayer {
136 layer: 0,
137 n_kv_heads,
138 head_dim,
139 });
140 }
141 let per_token = n_kv_heads * head_dim;
142 let tokens = measure_layer(0, first, per_token)?;
143
144 for (index, layer) in layers.iter().enumerate().skip(1) {
145 if layer.n_kv_heads != n_kv_heads || layer.head_dim != head_dim {
146 return Err(SignatureError::RaggedPayload {
147 layer: index,
148 field: "layer shape",
149 expected: format!("{n_kv_heads}x{head_dim}"),
150 found: format!("{}x{}", layer.n_kv_heads, layer.head_dim),
151 });
152 }
153 let layer_tokens = measure_layer(index, layer, per_token)?;
154 if layer_tokens != tokens {
155 return Err(SignatureError::RaggedPayload {
156 layer: index,
157 field: "token count",
158 expected: tokens.to_string(),
159 found: layer_tokens.to_string(),
160 });
161 }
162 }
163
164 if tokens != layout.block_size() {
167 return Err(SignatureError::BlockSizeMismatch {
168 block_size: layout.block_size(),
169 tokens,
170 });
171 }
172
173 Ok(CacheSignature {
174 format_version: BLOCK_FORMAT_VERSION,
175 model: model.to_string(),
176 n_layers: layers.len(),
177 n_kv_heads,
178 head_dim,
179 dtype: KvDtype::F32,
180 tokens,
181 layout,
182 })
183 }
184
185 pub fn expected(
188 model: &str,
189 layout: BlockLayout,
190 n_layers: usize,
191 n_kv_heads: usize,
192 head_dim: usize,
193 tokens: usize,
194 ) -> Self {
195 CacheSignature {
196 format_version: BLOCK_FORMAT_VERSION,
197 model: model.to_string(),
198 n_layers,
199 n_kv_heads,
200 head_dim,
201 dtype: KvDtype::F32,
202 tokens,
203 layout,
204 }
205 }
206
207 fn compare(
210 &self,
211 other: &CacheSignature,
212 mismatch: fn(&'static str, String, String) -> SignatureError,
213 ) -> Result<(), SignatureError> {
214 if self.format_version != other.format_version {
215 return Err(mismatch(
216 "format_version",
217 self.format_version.to_string(),
218 other.format_version.to_string(),
219 ));
220 }
221 if self.model != other.model {
222 return Err(mismatch("model", self.model.clone(), other.model.clone()));
223 }
224 if self.n_layers != other.n_layers {
225 return Err(mismatch(
226 "n_layers",
227 self.n_layers.to_string(),
228 other.n_layers.to_string(),
229 ));
230 }
231 if self.n_kv_heads != other.n_kv_heads {
232 return Err(mismatch(
233 "n_kv_heads",
234 self.n_kv_heads.to_string(),
235 other.n_kv_heads.to_string(),
236 ));
237 }
238 if self.head_dim != other.head_dim {
239 return Err(mismatch(
240 "head_dim",
241 self.head_dim.to_string(),
242 other.head_dim.to_string(),
243 ));
244 }
245 if self.dtype != other.dtype {
246 return Err(mismatch(
247 "dtype",
248 self.dtype.as_str().to_string(),
249 other.dtype.as_str().to_string(),
250 ));
251 }
252 if self.tokens != other.tokens {
253 return Err(mismatch(
254 "tokens",
255 self.tokens.to_string(),
256 other.tokens.to_string(),
257 ));
258 }
259 if self.layout.block_size() != other.layout.block_size() {
260 return Err(mismatch(
261 "block_size",
262 self.layout.block_size().to_string(),
263 other.layout.block_size().to_string(),
264 ));
265 }
266 if self.layout.sliding_window() != other.layout.sliding_window() {
267 return Err(mismatch(
268 "sliding_window",
269 describe_window(self.layout.sliding_window()),
270 describe_window(other.layout.sliding_window()),
271 ));
272 }
273 Ok(())
274 }
275}
276
277fn describe_window(window: Option<usize>) -> String {
281 match window {
282 Some(w) => w.to_string(),
283 None => "none (full causal)".to_string(),
284 }
285}
286
287fn measure_layer(index: usize, layer: &KvCache, per_token: usize) -> Result<usize, SignatureError> {
291 if !layer.k.len().is_multiple_of(per_token) {
292 return Err(SignatureError::RaggedPayload {
293 layer: index,
294 field: "k length",
295 expected: format!("a multiple of {per_token}"),
296 found: layer.k.len().to_string(),
297 });
298 }
299 if layer.v.len() != layer.k.len() {
300 return Err(SignatureError::RaggedPayload {
301 layer: index,
302 field: "v length",
303 expected: layer.k.len().to_string(),
304 found: layer.v.len().to_string(),
305 });
306 }
307 let tokens = layer.k.len() / per_token;
308 if layer.positions() != tokens {
316 return Err(SignatureError::RaggedPayload {
317 layer: index,
318 field: "positions",
319 expected: tokens.to_string(),
320 found: layer.positions().to_string(),
321 });
322 }
323 Ok(tokens)
324}
325
326#[derive(Clone, Debug, PartialEq, Eq)]
328pub enum SignatureError {
329 Unmarked,
333 EmptyPayload,
336 DegenerateLayer {
338 layer: usize,
339 n_kv_heads: usize,
340 head_dim: usize,
341 },
342 RaggedPayload {
346 layer: usize,
347 field: &'static str,
348 expected: String,
349 found: String,
350 },
351 PayloadMismatch {
355 field: &'static str,
356 recorded: String,
357 actual: String,
358 },
359 Incompatible {
363 field: &'static str,
364 expected: String,
365 found: String,
366 },
367 BlockSizeMismatch { block_size: usize, tokens: usize },
371 BadLayout(BlockLayoutError),
375 UnsupportedFormat {
377 found: u32,
378 readable: &'static [u32],
379 },
380}
381
382impl From<BlockLayoutError> for SignatureError {
383 fn from(err: BlockLayoutError) -> Self {
384 SignatureError::BadLayout(err)
385 }
386}
387
388impl std::fmt::Display for SignatureError {
389 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
390 match self {
391 SignatureError::Unmarked => write!(
392 f,
393 "KV block carries no cache signature; refusing to trust an unmarked block"
394 ),
395 SignatureError::EmptyPayload => {
396 write!(f, "KV block has no layers; nothing to verify")
397 }
398 SignatureError::DegenerateLayer {
399 layer,
400 n_kv_heads,
401 head_dim,
402 } => write!(
403 f,
404 "KV block layer {layer} is degenerate: {n_kv_heads} kv heads x {head_dim} head dim"
405 ),
406 SignatureError::RaggedPayload {
407 layer,
408 field,
409 expected,
410 found,
411 } => write!(
412 f,
413 "KV block payload is inconsistent at layer {layer}: {field} is {found}, expected {expected}"
414 ),
415 SignatureError::PayloadMismatch {
416 field,
417 recorded,
418 actual,
419 } => write!(
420 f,
421 "KV block signature vouches for {field}={recorded} but its payload has {field}={actual}"
422 ),
423 SignatureError::Incompatible {
424 field,
425 expected,
426 found,
427 } => write!(
428 f,
429 "KV block is incompatible: {field} is {found}, this server needs {expected}"
430 ),
431 SignatureError::BlockSizeMismatch { block_size, tokens } => write!(
432 f,
433 "KV block signature declares a block size of {block_size} but its payload holds \
434 {tokens} token positions; a stored block is exactly one whole block"
435 ),
436 SignatureError::BadLayout(err) => write!(f, "KV block layout is unusable: {err}"),
437 SignatureError::UnsupportedFormat { found, readable } => write!(
438 f,
439 "KV block format version {found} is not readable by this build (readable: {readable:?})"
440 ),
441 }
442 }
443}
444
445impl std::error::Error for SignatureError {}
446
447pub struct KvBlock {
452 signature: CacheSignature,
453 layers: Vec<KvCache>,
454}
455
456impl std::fmt::Debug for KvBlock {
460 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
461 f.debug_struct("KvBlock")
462 .field("signature", &self.signature)
463 .field("layers", &self.layers.len())
464 .finish()
465 }
466}
467
468impl std::fmt::Debug for UnverifiedBlock {
469 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
470 f.debug_struct("UnverifiedBlock")
471 .field("signature", &self.signature)
472 .field("layers", &self.layers.len())
473 .finish()
474 }
475}
476
477impl KvBlock {
478 pub fn stamp(
483 model: &str,
484 layout: BlockLayout,
485 layers: Vec<KvCache>,
486 ) -> Result<Self, SignatureError> {
487 let signature = CacheSignature::from_payload(model, layout, &layers)?;
488 Ok(KvBlock { signature, layers })
489 }
490
491 pub fn layout(&self) -> BlockLayout {
493 self.signature.layout
494 }
495
496 pub fn signature(&self) -> &CacheSignature {
497 &self.signature
498 }
499
500 pub fn tokens(&self) -> usize {
501 self.signature.tokens
502 }
503
504 pub fn layers(&self) -> &[KvCache] {
505 &self.layers
506 }
507
508 pub fn into_layers(self) -> Vec<KvCache> {
509 self.layers
510 }
511}
512
513pub struct UnverifiedBlock {
519 pub signature: Option<CacheSignature>,
520 pub layers: Vec<KvCache>,
521}
522
523impl UnverifiedBlock {
524 pub fn new(signature: Option<CacheSignature>, layers: Vec<KvCache>) -> Self {
525 UnverifiedBlock { signature, layers }
526 }
527
528 pub fn verify(self, expected: &CacheSignature) -> Result<KvBlock, SignatureError> {
532 let recorded = self.signature.ok_or(SignatureError::Unmarked)?;
533 if !READABLE_FORMAT_VERSIONS.contains(&recorded.format_version) {
534 return Err(SignatureError::UnsupportedFormat {
535 found: recorded.format_version,
536 readable: READABLE_FORMAT_VERSIONS,
537 });
538 }
539 let actual = CacheSignature::from_payload(&recorded.model, recorded.layout, &self.layers)?;
550 recorded.compare(&actual, |field, recorded, actual| {
551 SignatureError::PayloadMismatch {
552 field,
553 recorded,
554 actual,
555 }
556 })?;
557 expected.compare(&actual, |field, expected, found| {
558 SignatureError::Incompatible {
559 field,
560 expected,
561 found,
562 }
563 })?;
564 Ok(KvBlock {
565 signature: actual,
566 layers: self.layers,
567 })
568 }
569}
570
571#[cfg(test)]
572mod tests {
573 use super::*;
574
575 fn layer(n_kv_heads: usize, head_dim: usize, tokens: usize) -> KvCache {
576 let mut cache = KvCache::new(n_kv_heads, head_dim);
577 let step = vec![0.5f32; n_kv_heads * head_dim];
578 for _ in 0..tokens {
579 cache.push(&step, &step).expect("unpooled push cannot fail");
580 }
581 cache
582 }
583
584 fn payload(n_layers: usize, n_kv_heads: usize, head_dim: usize, tokens: usize) -> Vec<KvCache> {
585 (0..n_layers)
586 .map(|_| layer(n_kv_heads, head_dim, tokens))
587 .collect()
588 }
589
590 fn flat(block_size: usize) -> BlockLayout {
593 BlockLayout::full_attention(block_size).expect("positive block size")
594 }
595
596 #[test]
599 fn signature_is_measured_from_the_payload() {
600 let block = KvBlock::stamp("model-a", flat(4), payload(3, 2, 8, 4)).expect("stamp");
601 let sig = block.signature();
602 assert_eq!(sig.n_layers, 3);
603 assert_eq!(sig.n_kv_heads, 2);
604 assert_eq!(sig.head_dim, 8);
605 assert_eq!(sig.tokens, 4);
606 assert_eq!(sig.dtype, KvDtype::F32);
607 assert_eq!(sig.format_version, BLOCK_FORMAT_VERSION);
608 assert_eq!(block.tokens(), 4);
609 assert_eq!(block.layers().len(), 3);
610 }
611
612 #[test]
613 fn a_stamped_block_round_trips_through_verification() {
614 let layers = payload(3, 2, 8, 4);
615 let signature =
616 CacheSignature::from_payload("model-a", flat(4), &layers).expect("signature");
617 let expected = CacheSignature::expected("model-a", flat(4), 3, 2, 8, 4);
618 let block = UnverifiedBlock::new(Some(signature), layers)
619 .verify(&expected)
620 .expect("a block that is what it says it is must verify");
621 assert_eq!(block.layers().len(), 3);
622 assert_eq!(block.into_layers().len(), 3);
623 }
624
625 #[test]
630 fn an_unmarked_block_is_rejected_not_trusted() {
631 let expected = CacheSignature::expected("model-a", flat(4), 3, 2, 8, 4);
632 let err = UnverifiedBlock::new(None, payload(3, 2, 8, 4))
633 .verify(&expected)
634 .expect_err("an unmarked block must be refused");
635 assert_eq!(err, SignatureError::Unmarked);
636 }
637
638 #[test]
645 fn a_signature_that_overstates_its_payload_is_rejected() {
646 let expected = CacheSignature::expected("model-a", flat(4), 3, 2, 16, 4);
647 let mut lying = expected.clone();
648 assert_eq!(lying.head_dim, 16);
649 let err = UnverifiedBlock::new(Some(lying.clone()), payload(3, 2, 8, 4))
650 .verify(&expected)
651 .expect_err("stamp claims head_dim 16 over an 8-wide payload");
652 assert_eq!(
653 err,
654 SignatureError::PayloadMismatch {
655 field: "head_dim",
656 recorded: "16".into(),
657 actual: "8".into(),
658 }
659 );
660
661 lying.head_dim = 8;
664 lying.tokens = 8;
665 let expected = CacheSignature::expected("model-a", flat(8), 3, 2, 8, 8);
666 let err = UnverifiedBlock::new(Some(lying.clone()), payload(3, 2, 8, 4))
667 .verify(&expected)
668 .expect_err("stamp claims 8 tokens over a 4-token payload");
669 assert_eq!(
670 err,
671 SignatureError::PayloadMismatch {
672 field: "tokens",
673 recorded: "8".into(),
674 actual: "4".into(),
675 }
676 );
677
678 lying.tokens = 4;
680 lying.n_layers = 4;
681 let expected = CacheSignature::expected("model-a", flat(4), 4, 2, 8, 4);
682 let err = UnverifiedBlock::new(Some(lying), payload(3, 2, 8, 4))
683 .verify(&expected)
684 .expect_err("stamp claims 4 layers over a 3-layer payload");
685 assert_eq!(
686 err,
687 SignatureError::PayloadMismatch {
688 field: "n_layers",
689 recorded: "4".into(),
690 actual: "3".into(),
691 }
692 );
693 }
694
695 #[test]
698 fn a_layer_whose_seq_len_contradicts_its_buffers_is_rejected() {
699 let mut layers = payload(2, 2, 8, 4);
700 layers[1].force_positions_for_test(7);
702 let err = CacheSignature::from_payload("model-a", flat(4), &layers)
703 .expect_err("seq_len must be verified, not believed");
704 assert_eq!(
705 err,
706 SignatureError::RaggedPayload {
707 layer: 1,
708 field: "positions",
709 expected: "4".into(),
710 found: "7".into(),
711 }
712 );
713 }
714
715 #[test]
716 fn a_ragged_payload_is_rejected() {
717 let mut layers = payload(3, 2, 8, 4);
718 layers[2] = layer(2, 4, 4);
719 let err = CacheSignature::from_payload("model-a", flat(4), &layers)
720 .expect_err("shape disagreement");
721 assert!(matches!(
722 err,
723 SignatureError::RaggedPayload {
724 layer: 2,
725 field: "layer shape",
726 ..
727 }
728 ));
729
730 let mut layers = payload(3, 2, 8, 4);
731 layers[1] = layer(2, 8, 3);
732 let err = CacheSignature::from_payload("model-a", flat(4), &layers)
733 .expect_err("depth disagreement");
734 assert!(matches!(
735 err,
736 SignatureError::RaggedPayload {
737 layer: 1,
738 field: "token count",
739 ..
740 }
741 ));
742
743 let mut layers = payload(2, 2, 8, 4);
744 layers[0].v.truncate(8);
745 let err = CacheSignature::from_payload("model-a", flat(4), &layers)
746 .expect_err("k/v disagreement");
747 assert!(matches!(
748 err,
749 SignatureError::RaggedPayload {
750 layer: 0,
751 field: "v length",
752 ..
753 }
754 ));
755 }
756
757 #[test]
758 fn an_empty_payload_is_rejected() {
759 assert_eq!(
760 CacheSignature::from_payload("model-a", flat(4), &[])
761 .expect_err("nothing to vouch for"),
762 SignatureError::EmptyPayload
763 );
764 }
765
766 #[test]
770 fn an_honest_block_from_a_different_config_is_incompatible() {
771 let layers = payload(3, 2, 8, 4);
772 let signature =
773 CacheSignature::from_payload("model-a", flat(4), &layers).expect("signature");
774 let err = UnverifiedBlock::new(Some(signature.clone()), layers)
775 .verify(&CacheSignature::expected("model-b", flat(4), 3, 2, 8, 4))
776 .expect_err("a different model must not share KV state");
777 assert_eq!(
778 err,
779 SignatureError::Incompatible {
780 field: "model",
781 expected: "model-b".into(),
782 found: "model-a".into(),
783 }
784 );
785
786 let layers = payload(3, 2, 8, 4);
787 let err = UnverifiedBlock::new(Some(signature), layers)
788 .verify(&CacheSignature::expected("model-a", flat(4), 3, 4, 8, 4))
789 .expect_err("a different KV head count must not be reused");
790 assert_eq!(
791 err,
792 SignatureError::Incompatible {
793 field: "n_kv_heads",
794 expected: "4".into(),
795 found: "2".into(),
796 }
797 );
798 }
799
800 #[test]
801 fn an_unreadable_format_version_is_rejected() {
802 let layers = payload(2, 2, 8, 4);
803 let mut signature =
804 CacheSignature::from_payload("model-a", flat(4), &layers).expect("signature");
805 signature.format_version = 99;
806 let err = UnverifiedBlock::new(Some(signature), layers)
807 .verify(&CacheSignature::expected("model-a", flat(4), 2, 2, 8, 4))
808 .expect_err("an unknown layout must not be guessed at");
809 assert_eq!(
810 err,
811 SignatureError::UnsupportedFormat {
812 found: 99,
813 readable: READABLE_FORMAT_VERSIONS,
814 }
815 );
816 }
817
818 #[test]
826 fn a_block_written_under_a_different_window_is_refused_not_reused() {
827 let layout_128 = BlockLayout::new(4, Some(128)).expect("4 divides 128");
828 let layout_256 = BlockLayout::new(4, Some(256)).expect("4 divides 256");
829 let layers = payload(3, 2, 8, 4);
830 let signature =
831 CacheSignature::from_payload("model-a", layout_128, &layers).expect("signature");
832
833 let err = UnverifiedBlock::new(Some(signature.clone()), layers)
834 .verify(&CacheSignature::expected("model-a", layout_256, 3, 2, 8, 4))
835 .expect_err("a window change must invalidate the block, not be ignored");
836 assert_eq!(
837 err,
838 SignatureError::Incompatible {
839 field: "sliding_window",
840 expected: "256".into(),
841 found: "128".into(),
842 }
843 );
844
845 let layers = payload(3, 2, 8, 4);
848 UnverifiedBlock::new(Some(signature), layers)
849 .verify(&CacheSignature::expected("model-a", layout_128, 3, 2, 8, 4))
850 .expect("unchanged config must still hit");
851 }
852
853 #[test]
856 fn a_full_causal_reader_will_not_take_a_sliding_window_block() {
857 let sliding = BlockLayout::new(4, Some(128)).expect("aligned");
858 let layers = payload(2, 2, 8, 4);
859 let signature =
860 CacheSignature::from_payload("model-a", sliding, &layers).expect("signature");
861 let err = UnverifiedBlock::new(Some(signature), layers)
862 .verify(&CacheSignature::expected("model-a", flat(4), 2, 2, 8, 4))
863 .expect_err("no window and a 128 window are different configurations");
864 assert_eq!(
865 err,
866 SignatureError::Incompatible {
867 field: "sliding_window",
868 expected: "none (full causal)".into(),
869 found: "128".into(),
870 }
871 );
872 }
873
874 #[test]
878 fn a_block_cut_at_a_different_block_size_is_incompatible() {
879 let layers = payload(2, 2, 8, 4);
880 let signature =
881 CacheSignature::from_payload("model-a", flat(4), &layers).expect("signature");
882 let err = UnverifiedBlock::new(Some(signature), layers)
883 .verify(&CacheSignature::expected("model-a", flat(2), 2, 2, 8, 4))
884 .expect_err("a 4-token block is not a 2-token block");
885 assert_eq!(
886 err,
887 SignatureError::Incompatible {
888 field: "block_size",
889 expected: "2".into(),
890 found: "4".into(),
891 }
892 );
893 }
894
895 #[test]
899 fn a_stamp_may_not_claim_a_block_size_the_payload_lacks() {
900 let err = KvBlock::stamp("model-a", flat(8), payload(2, 2, 8, 4))
901 .expect_err("8-token blocks over a 4-token payload");
902 assert_eq!(
903 err,
904 SignatureError::BlockSizeMismatch {
905 block_size: 8,
906 tokens: 4,
907 }
908 );
909
910 let honest =
913 CacheSignature::from_payload("model-a", flat(4), &payload(2, 2, 8, 4)).expect("sig");
914 let mut lying = honest.clone();
915 lying.layout = flat(8);
916 lying.tokens = 8;
917 let err = UnverifiedBlock::new(Some(lying), payload(2, 2, 8, 4))
918 .verify(&CacheSignature::expected("model-a", flat(8), 2, 2, 8, 8))
919 .expect_err("the payload settles the block size, not the stamp");
920 assert_eq!(
921 err,
922 SignatureError::BlockSizeMismatch {
923 block_size: 8,
924 tokens: 4,
925 }
926 );
927 }
928
929 #[test]
933 fn blocks_from_the_pre_layout_format_are_not_readable() {
934 assert!(!READABLE_FORMAT_VERSIONS.contains(&1));
935 let layers = payload(2, 2, 8, 4);
936 let mut signature =
937 CacheSignature::from_payload("model-a", flat(4), &layers).expect("signature");
938 signature.format_version = 1;
939 let err = UnverifiedBlock::new(Some(signature), layers)
940 .verify(&CacheSignature::expected("model-a", flat(4), 2, 2, 8, 4))
941 .expect_err("a v1 block cannot say what layout it was cut under");
942 assert_eq!(
943 err,
944 SignatureError::UnsupportedFormat {
945 found: 1,
946 readable: READABLE_FORMAT_VERSIONS,
947 }
948 );
949 }
950
951 #[test]
952 fn errors_name_the_field_that_changed() {
953 let text = SignatureError::Incompatible {
954 field: "head_dim",
955 expected: "128".into(),
956 found: "64".into(),
957 }
958 .to_string();
959 assert!(text.contains("head_dim"), "{text}");
960 assert!(text.contains("64"), "{text}");
961 assert!(text.contains("128"), "{text}");
962 assert!(SignatureError::Unmarked.to_string().contains("unmarked"));
963 }
964}