1#[cfg(feature = "parallel")]
11#[allow(unused_imports)]
12use rayon::prelude::*;
13use scirs2_core::ndarray::{s, Array1, Array2};
14use sklears_core::{
15 error::{Result, SklearsError},
16 types::Float,
17};
18use std::convert::TryFrom;
19
20#[derive(Debug, Clone)]
22pub struct CompressionConfig {
23 pub method: CompressionMethod,
25 pub compression_ratio: Float,
27 pub quality_threshold: Float,
29 pub num_components: Option<usize>,
31 pub quantization_levels: usize,
33 pub block_size: usize,
35 pub adaptive: bool,
37}
38
39impl Default for CompressionConfig {
40 fn default() -> Self {
41 Self {
42 method: CompressionMethod::LowRank,
43 compression_ratio: 0.1,
44 quality_threshold: 1e-6,
45 num_components: None,
46 quantization_levels: 256,
47 block_size: 1000,
48 adaptive: true,
49 }
50 }
51}
52
53#[derive(Debug, Clone, Copy)]
55pub enum CompressionMethod {
56 LowRank,
58 Quantized,
60 Sparse,
62 Hierarchical,
64 Adaptive,
66 Nystrom,
68 RandomFourier,
70}
71
72pub trait CompressedKernelMatrix: Send + Sync {
74 fn get(&self, i: usize, j: usize) -> Result<Float>;
76
77 fn get_block(
79 &self,
80 row_start: usize,
81 row_end: usize,
82 col_start: usize,
83 col_end: usize,
84 ) -> Result<Array2<Float>>;
85
86 fn dimensions(&self) -> (usize, usize);
88
89 fn compression_ratio(&self) -> Float;
91
92 fn approximation_error(&self) -> Float;
94
95 fn memory_usage(&self) -> usize;
97}
98
99pub struct LowRankKernelMatrix {
101 left_factors: Array2<Float>,
102 right_factors: Array2<Float>,
103 #[allow(dead_code)] singular_values: Array1<Float>,
105 dimensions: (usize, usize),
106 approximation_error: Float,
107 original_memory: usize,
108}
109
110impl LowRankKernelMatrix {
111 pub fn new(kernel_matrix: &Array2<Float>, num_components: usize) -> Result<Self> {
113 let dimensions = kernel_matrix.dim();
114 let original_memory = dimensions.0 * dimensions.1 * std::mem::size_of::<Float>();
115
116 let (mut u, mut s, mut vt) = Self::svd_decomposition(kernel_matrix, num_components)?;
118
119 let (m, n) = dimensions;
121 let mut effective_components = s.len();
122 let element_size = std::mem::size_of::<Float>();
123 let mut compressed_memory =
124 (m * effective_components + n * effective_components) * element_size;
125
126 while effective_components > 1 && compressed_memory >= original_memory {
127 effective_components -= 1;
128 compressed_memory =
129 (m * effective_components + n * effective_components) * element_size;
130 }
131
132 if effective_components != s.len() {
133 u = u.slice(s![.., ..effective_components]).to_owned();
134 s = s.slice(s![..effective_components]).to_owned();
135 vt = vt.slice(s![..effective_components, ..]).to_owned();
136 }
137
138 let approximation_error = Self::compute_approximation_error(kernel_matrix, &u, &s, &vt)?;
140
141 let mut left_factors = u;
143 let mut right_factors = vt;
144 for (idx, singular) in s.iter().enumerate() {
145 if *singular <= 0.0 {
146 left_factors.column_mut(idx).fill(0.0);
147 right_factors.row_mut(idx).fill(0.0);
148 } else {
149 let scale = singular.sqrt();
150 left_factors.column_mut(idx).mapv_inplace(|val| val * scale);
151 right_factors.row_mut(idx).mapv_inplace(|val| val * scale);
152 }
153 }
154
155 Ok(Self {
156 left_factors,
157 right_factors,
158 singular_values: s,
159 dimensions,
160 approximation_error,
161 original_memory,
162 })
163 }
164
165 fn svd_decomposition(
167 matrix: &Array2<Float>,
168 k: usize,
169 ) -> Result<(Array2<Float>, Array1<Float>, Array2<Float>)> {
170 let (m, n) = matrix.dim();
173 let rank = k.min(m).min(n);
174
175 let ata = matrix.t().dot(matrix);
177 let (eigenvals, eigenvecs) = Self::compute_eigenpairs(&ata, rank)?;
178
179 let mut s = Array1::zeros(rank);
181 for i in 0..rank {
182 s[i] = eigenvals[i].sqrt();
183 }
184
185 let mut u = Array2::zeros((m, rank));
187 for i in 0..rank {
188 if s[i] > 1e-12 {
189 let v_col = eigenvecs.column(i);
190 let u_col = matrix.dot(&v_col) / s[i];
191 u.column_mut(i).assign(&u_col);
192 }
193 }
194
195 let vt = eigenvecs.t().to_owned();
197
198 Ok((u, s, vt.slice(s![..rank, ..]).to_owned()))
199 }
200
201 fn compute_eigenpairs(
203 matrix: &Array2<Float>,
204 k: usize,
205 ) -> Result<(Array1<Float>, Array2<Float>)> {
206 let n = matrix.nrows();
207 let mut eigenvals = Array1::zeros(k);
208 let mut eigenvecs = Array2::zeros((n, k));
209
210 for i in 0..k {
212 let mut v = Array1::from_elem(n, 1.0 / (n as Float).sqrt());
213
214 for j in 0..i {
216 let proj = v.dot(&eigenvecs.column(j));
217 v = &v - &(&eigenvecs.column(j).to_owned() * proj);
218 }
219
220 for _ in 0..100 {
222 let mut v_new = matrix.dot(&v);
223
224 for j in 0..i {
226 let proj = v_new.dot(&eigenvecs.column(j));
227 v_new = &v_new - &(&eigenvecs.column(j).to_owned() * proj);
228 }
229
230 let norm = v_new.iter().map(|x| x * x).sum::<Float>().sqrt();
231 if norm < 1e-12 {
232 break;
233 }
234 v_new /= norm;
235
236 if (&v_new - &v).iter().map(|x| x.abs()).sum::<Float>() < 1e-8 {
237 break;
238 }
239 v = v_new;
240 }
241
242 let eigenval = v.dot(&matrix.dot(&v));
243 eigenvals[i] = eigenval;
244 eigenvecs.column_mut(i).assign(&v);
245 }
246
247 Ok((eigenvals, eigenvecs))
248 }
249
250 fn compute_approximation_error(
252 original: &Array2<Float>,
253 u: &Array2<Float>,
254 s: &Array1<Float>,
255 vt: &Array2<Float>,
256 ) -> Result<Float> {
257 let (m, n) = original.dim();
258 let mut total_error = 0.0;
259 let mut count = 0;
260
261 for i in (0..m).step_by(m / 100 + 1) {
263 for j in (0..n).step_by(n / 100 + 1) {
264 let original_val = original[[i, j]];
265 let mut approx_val = 0.0;
266
267 for k in 0..u.ncols() {
268 approx_val += u[[i, k]] * s[k] * vt[[k, j]];
269 }
270
271 total_error += (original_val - approx_val).powi(2);
272 count += 1;
273 }
274 }
275
276 Ok((total_error / count as Float).sqrt())
277 }
278}
279
280impl CompressedKernelMatrix for LowRankKernelMatrix {
281 fn get(&self, i: usize, j: usize) -> Result<Float> {
282 if i >= self.dimensions.0 || j >= self.dimensions.1 {
283 return Err(SklearsError::InvalidInput(
284 "Index out of bounds".to_string(),
285 ));
286 }
287
288 let mut value = 0.0;
289 for k in 0..self.left_factors.ncols() {
290 value += self.left_factors[[i, k]] * self.right_factors[[k, j]];
291 }
292
293 Ok(value)
294 }
295
296 fn get_block(
297 &self,
298 row_start: usize,
299 row_end: usize,
300 col_start: usize,
301 col_end: usize,
302 ) -> Result<Array2<Float>> {
303 let block_rows = row_end - row_start;
304 let block_cols = col_end - col_start;
305 let mut block = Array2::zeros((block_rows, block_cols));
306
307 let u_block = self.left_factors.slice(s![row_start..row_end, ..]);
308 let vt_block = self.right_factors.slice(s![.., col_start..col_end]);
309
310 for k in 0..self.left_factors.ncols() {
311 let u_col = u_block.column(k);
312 let vt_row = vt_block.row(k);
313
314 for i in 0..block_rows {
315 for j in 0..block_cols {
316 block[[i, j]] += u_col[i] * vt_row[j];
317 }
318 }
319 }
320
321 Ok(block)
322 }
323
324 fn dimensions(&self) -> (usize, usize) {
325 self.dimensions
326 }
327
328 fn compression_ratio(&self) -> Float {
329 let compressed_memory =
330 (self.left_factors.len() + self.right_factors.len()) * std::mem::size_of::<Float>();
331 compressed_memory as Float / self.original_memory as Float
332 }
333
334 fn approximation_error(&self) -> Float {
335 self.approximation_error
336 }
337
338 fn memory_usage(&self) -> usize {
339 (self.left_factors.len() + self.right_factors.len()) * std::mem::size_of::<Float>()
340 }
341}
342
343pub struct QuantizedKernelMatrix {
345 quantized_data: Vec<u8>,
346 min_value: Float,
347 max_value: Float,
348 quantization_levels: usize,
349 dimensions: (usize, usize),
350 original_memory: usize,
351}
352
353impl QuantizedKernelMatrix {
354 pub fn new(kernel_matrix: &Array2<Float>, quantization_levels: usize) -> Result<Self> {
356 let dimensions = kernel_matrix.dim();
357 let original_memory = dimensions.0 * dimensions.1 * std::mem::size_of::<Float>();
358
359 let min_value = kernel_matrix.iter().fold(Float::INFINITY, |a, &b| a.min(b));
360 let max_value = kernel_matrix
361 .iter()
362 .fold(Float::NEG_INFINITY, |a, &b| a.max(b));
363
364 let range = max_value - min_value;
365 let scale = (quantization_levels - 1) as Float / range;
366
367 let quantized_data: Vec<u8> = kernel_matrix
368 .iter()
369 .map(|&val| {
370 let normalized = (val - min_value) * scale;
371 normalized
372 .round()
373 .min((quantization_levels - 1) as Float)
374 .max(0.0) as u8
375 })
376 .collect();
377
378 Ok(Self {
379 quantized_data,
380 min_value,
381 max_value,
382 quantization_levels,
383 dimensions,
384 original_memory,
385 })
386 }
387
388 fn dequantize(&self, quantized: u8) -> Float {
390 let range = self.max_value - self.min_value;
391 let scale = range / (self.quantization_levels - 1) as Float;
392 self.min_value + quantized as Float * scale
393 }
394}
395
396impl CompressedKernelMatrix for QuantizedKernelMatrix {
397 fn get(&self, i: usize, j: usize) -> Result<Float> {
398 if i >= self.dimensions.0 || j >= self.dimensions.1 {
399 return Err(SklearsError::InvalidInput(
400 "Index out of bounds".to_string(),
401 ));
402 }
403
404 let idx = i * self.dimensions.1 + j;
405 let quantized = self.quantized_data[idx];
406 Ok(self.dequantize(quantized))
407 }
408
409 fn get_block(
410 &self,
411 row_start: usize,
412 row_end: usize,
413 col_start: usize,
414 col_end: usize,
415 ) -> Result<Array2<Float>> {
416 let block_rows = row_end - row_start;
417 let block_cols = col_end - col_start;
418 let mut block = Array2::zeros((block_rows, block_cols));
419
420 for i in 0..block_rows {
421 for j in 0..block_cols {
422 let global_i = row_start + i;
423 let global_j = col_start + j;
424 let idx = global_i * self.dimensions.1 + global_j;
425 block[[i, j]] = self.dequantize(self.quantized_data[idx]);
426 }
427 }
428
429 Ok(block)
430 }
431
432 fn dimensions(&self) -> (usize, usize) {
433 self.dimensions
434 }
435
436 fn compression_ratio(&self) -> Float {
437 let compressed_memory = self.quantized_data.len() * std::mem::size_of::<u8>() +
438 2 * std::mem::size_of::<Float>() + std::mem::size_of::<usize>(); compressed_memory as Float / self.original_memory as Float
441 }
442
443 fn approximation_error(&self) -> Float {
444 let range = self.max_value - self.min_value;
445 range / (self.quantization_levels - 1) as Float / 2.0 }
447
448 fn memory_usage(&self) -> usize {
449 self.quantized_data.len() * std::mem::size_of::<u8>()
450 + 2 * std::mem::size_of::<Float>()
451 + std::mem::size_of::<usize>()
452 }
453}
454
455pub struct SparseKernelMatrix {
457 values: Vec<Float>,
458 row_indices: Vec<u32>,
459 col_indices: Vec<u32>,
460 dimensions: (usize, usize),
461 sparsity_threshold: Float,
462 original_memory: usize,
463}
464
465impl SparseKernelMatrix {
466 pub fn new(kernel_matrix: &Array2<Float>, sparsity_threshold: Float) -> Result<Self> {
468 let dimensions = kernel_matrix.dim();
469 let original_memory = dimensions.0 * dimensions.1 * std::mem::size_of::<Float>();
470
471 let mut values = Vec::new();
472 let mut row_indices = Vec::new();
473 let mut col_indices = Vec::new();
474
475 for i in 0..dimensions.0 {
476 for j in 0..dimensions.1 {
477 let val = kernel_matrix[[i, j]];
478 if val.abs() >= sparsity_threshold {
479 values.push(val);
480 row_indices.push(u32::try_from(i).map_err(|_| {
481 SklearsError::InvalidInput(
482 "Sparse kernel row index exceeds 32-bit storage".to_string(),
483 )
484 })?);
485 col_indices.push(u32::try_from(j).map_err(|_| {
486 SklearsError::InvalidInput(
487 "Sparse kernel column index exceeds 32-bit storage".to_string(),
488 )
489 })?);
490 }
491 }
492 }
493
494 Ok(Self {
495 values,
496 row_indices,
497 col_indices,
498 dimensions,
499 sparsity_threshold,
500 original_memory,
501 })
502 }
503
504 fn find_index(&self, i: usize, j: usize) -> Option<usize> {
506 for (idx, (&row, &col)) in self.row_indices.iter().zip(&self.col_indices).enumerate() {
507 if row as usize == i && col as usize == j {
508 return Some(idx);
509 }
510 }
511 None
512 }
513}
514
515impl CompressedKernelMatrix for SparseKernelMatrix {
516 fn get(&self, i: usize, j: usize) -> Result<Float> {
517 if i >= self.dimensions.0 || j >= self.dimensions.1 {
518 return Err(SklearsError::InvalidInput(
519 "Index out of bounds".to_string(),
520 ));
521 }
522
523 if let Some(idx) = self.find_index(i, j) {
524 Ok(self.values[idx])
525 } else {
526 Ok(0.0) }
528 }
529
530 fn get_block(
531 &self,
532 row_start: usize,
533 row_end: usize,
534 col_start: usize,
535 col_end: usize,
536 ) -> Result<Array2<Float>> {
537 let block_rows = row_end - row_start;
538 let block_cols = col_end - col_start;
539 let mut block = Array2::zeros((block_rows, block_cols));
540
541 for (idx, (&row, &col)) in self.row_indices.iter().zip(&self.col_indices).enumerate() {
542 let row = row as usize;
543 let col = col as usize;
544 if row >= row_start && row < row_end && col >= col_start && col < col_end {
545 block[[row - row_start, col - col_start]] = self.values[idx];
546 }
547 }
548
549 Ok(block)
550 }
551
552 fn dimensions(&self) -> (usize, usize) {
553 self.dimensions
554 }
555
556 fn compression_ratio(&self) -> Float {
557 let compressed_memory = self.values.len() * std::mem::size_of::<Float>()
558 + self.row_indices.len() * std::mem::size_of::<u32>()
559 + self.col_indices.len() * std::mem::size_of::<u32>();
560 compressed_memory as Float / self.original_memory as Float
561 }
562
563 fn approximation_error(&self) -> Float {
564 self.sparsity_threshold
565 }
566
567 fn memory_usage(&self) -> usize {
568 self.values.len() * std::mem::size_of::<Float>()
569 + self.row_indices.len() * std::mem::size_of::<u32>()
570 + self.col_indices.len() * std::mem::size_of::<u32>()
571 }
572}
573
574pub struct HierarchicalKernelMatrix {
576 blocks: Vec<Vec<Box<dyn CompressedKernelMatrix>>>,
577 block_size: usize,
578 dimensions: (usize, usize),
579 #[allow(dead_code)] compression_config: CompressionConfig,
581}
582
583impl HierarchicalKernelMatrix {
584 pub fn new(kernel_matrix: &Array2<Float>, config: CompressionConfig) -> Result<Self> {
586 let dimensions = kernel_matrix.dim();
587 let block_size = config.block_size;
588
589 let num_block_rows = dimensions.0.div_ceil(block_size);
590 let num_block_cols = dimensions.1.div_ceil(block_size);
591
592 let mut blocks = Vec::with_capacity(num_block_rows);
593
594 for i in 0..num_block_rows {
595 let mut block_row = Vec::with_capacity(num_block_cols);
596
597 for j in 0..num_block_cols {
598 let row_start = i * block_size;
599 let row_end = ((i + 1) * block_size).min(dimensions.0);
600 let col_start = j * block_size;
601 let col_end = ((j + 1) * block_size).min(dimensions.1);
602
603 let block = kernel_matrix.slice(s![row_start..row_end, col_start..col_end]);
604 let compressed_block = Self::compress_block(&block.to_owned(), &config)?;
605 block_row.push(compressed_block);
606 }
607
608 blocks.push(block_row);
609 }
610
611 Ok(Self {
612 blocks,
613 block_size,
614 dimensions,
615 compression_config: config,
616 })
617 }
618
619 fn compress_block(
621 block: &Array2<Float>,
622 config: &CompressionConfig,
623 ) -> Result<Box<dyn CompressedKernelMatrix>> {
624 match config.method {
625 CompressionMethod::LowRank => {
626 let k = config
627 .num_components
628 .unwrap_or(block.nrows().min(block.ncols()) / 4);
629 Ok(Box::new(LowRankKernelMatrix::new(block, k)?))
630 }
631 CompressionMethod::Quantized => Ok(Box::new(QuantizedKernelMatrix::new(
632 block,
633 config.quantization_levels,
634 )?)),
635 CompressionMethod::Sparse => Ok(Box::new(SparseKernelMatrix::new(
636 block,
637 config.quality_threshold,
638 )?)),
639 CompressionMethod::Adaptive => {
640 Self::adaptive_compression(block, config)
642 }
643 _ => {
644 let k = config
646 .num_components
647 .unwrap_or(block.nrows().min(block.ncols()) / 4);
648 Ok(Box::new(LowRankKernelMatrix::new(block, k)?))
649 }
650 }
651 }
652
653 fn adaptive_compression(
655 block: &Array2<Float>,
656 config: &CompressionConfig,
657 ) -> Result<Box<dyn CompressedKernelMatrix>> {
658 let methods = [
660 CompressionMethod::LowRank,
661 CompressionMethod::Quantized,
662 CompressionMethod::Sparse,
663 ];
664
665 let mut best_compression: Option<Box<dyn CompressedKernelMatrix>> = None;
666 let mut best_score = Float::NEG_INFINITY;
667
668 for method in &methods {
669 let mut test_config = config.clone();
670 test_config.method = *method;
671
672 if let Ok(compressed) = Self::compress_block(block, &test_config) {
673 let compression_ratio = compressed.compression_ratio();
675 let error = compressed.approximation_error();
676 let score = compression_ratio - error * 10.0; if score > best_score {
679 best_score = score;
680 best_compression = Some(compressed);
681 }
682 }
683 }
684
685 best_compression.ok_or_else(|| {
686 SklearsError::InvalidInput("No compression method succeeded".to_string())
687 })
688 }
689
690 fn get_block_indices(&self, i: usize, j: usize) -> (usize, usize, usize, usize) {
692 let block_i = i / self.block_size;
693 let block_j = j / self.block_size;
694 let local_i = i % self.block_size;
695 let local_j = j % self.block_size;
696 (block_i, block_j, local_i, local_j)
697 }
698}
699
700impl CompressedKernelMatrix for HierarchicalKernelMatrix {
701 fn get(&self, i: usize, j: usize) -> Result<Float> {
702 if i >= self.dimensions.0 || j >= self.dimensions.1 {
703 return Err(SklearsError::InvalidInput(
704 "Index out of bounds".to_string(),
705 ));
706 }
707
708 let (block_i, block_j, local_i, local_j) = self.get_block_indices(i, j);
709 self.blocks[block_i][block_j].get(local_i, local_j)
710 }
711
712 fn get_block(
713 &self,
714 row_start: usize,
715 row_end: usize,
716 col_start: usize,
717 col_end: usize,
718 ) -> Result<Array2<Float>> {
719 let block_rows = row_end - row_start;
720 let block_cols = col_end - col_start;
721 let mut result = Array2::zeros((block_rows, block_cols));
722
723 for i in 0..block_rows {
724 for j in 0..block_cols {
725 result[[i, j]] = self.get(row_start + i, col_start + j)?;
726 }
727 }
728
729 Ok(result)
730 }
731
732 fn dimensions(&self) -> (usize, usize) {
733 self.dimensions
734 }
735
736 fn compression_ratio(&self) -> Float {
737 let total_memory: usize = self
738 .blocks
739 .iter()
740 .flat_map(|row| row.iter())
741 .map(|block| block.memory_usage())
742 .sum();
743
744 let original_memory = self.dimensions.0 * self.dimensions.1 * std::mem::size_of::<Float>();
745 total_memory as Float / original_memory as Float
746 }
747
748 fn approximation_error(&self) -> Float {
749 let total_error: Float = self
751 .blocks
752 .iter()
753 .flat_map(|row| row.iter())
754 .map(|block| block.approximation_error())
755 .sum();
756
757 let num_blocks = self.blocks.len() * self.blocks[0].len();
758 total_error / num_blocks as Float
759 }
760
761 fn memory_usage(&self) -> usize {
762 self.blocks
763 .iter()
764 .flat_map(|row| row.iter())
765 .map(|block| block.memory_usage())
766 .sum()
767 }
768}
769
770#[allow(non_snake_case)]
771#[cfg(test)]
772mod tests {
773 use super::*;
774
775 #[test]
776 fn test_low_rank_compression() {
777 let matrix = Array2::from_shape_vec(
778 (4, 4),
779 vec![
780 1.0, 0.8, 0.6, 0.4, 0.8, 1.0, 0.7, 0.5, 0.6, 0.7, 1.0, 0.6, 0.4, 0.5, 0.6, 1.0,
781 ],
782 )
783 .expect("operation should succeed");
784
785 let compressed = LowRankKernelMatrix::new(&matrix, 2).expect("construction should succeed");
786 assert_eq!(compressed.dimensions(), (4, 4));
787 assert!(compressed.compression_ratio() < 1.0);
788 }
789
790 #[test]
791 fn test_quantized_compression() {
792 let matrix =
793 Array2::from_shape_vec((3, 3), vec![1.0, 0.5, 0.0, 0.5, 1.0, 0.5, 0.0, 0.5, 1.0])
794 .expect("operation should succeed");
795
796 let compressed =
797 QuantizedKernelMatrix::new(&matrix, 16).expect("construction should succeed");
798 assert_eq!(compressed.dimensions(), (3, 3));
799 assert!(compressed.compression_ratio() < 1.0);
800 }
801
802 #[test]
803 fn test_sparse_compression() {
804 let matrix = Array2::from_shape_vec(
805 (4, 4),
806 vec![
807 1.0, 0.0, 0.0, 0.8, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.8, 0.0, 0.0, 1.0,
808 ],
809 )
810 .expect("operation should succeed");
811
812 let compressed =
813 SparseKernelMatrix::new(&matrix, 0.1).expect("construction should succeed");
814 assert_eq!(compressed.dimensions(), (4, 4));
815 assert!(compressed.compression_ratio() < 1.0);
816 }
817
818 #[test]
819 fn test_hierarchical_compression() {
820 let matrix = Array2::eye(8);
821 let config = CompressionConfig {
822 block_size: 4,
823 method: CompressionMethod::LowRank,
824 ..CompressionConfig::default()
825 };
826
827 let compressed =
828 HierarchicalKernelMatrix::new(&matrix, config).expect("construction should succeed");
829 assert_eq!(compressed.dimensions(), (8, 8));
830 }
831}