1use crate::{FloatElement, Tensor, TensorElement};
18use scirs2_core::ndarray::{Array1, Array2};
19use scirs2_core::numeric::ToPrimitive;
20use std::collections::HashMap;
21use torsh_core::error::{Result, TorshError};
22
23pub struct SciRS2StatsProcessor {
25 config: StatsConfig,
26}
27
28#[derive(Debug, Clone)]
30pub struct StatsConfig {
31 pub confidence_level: f64,
33 pub missing_value_strategy: MissingValueStrategy,
35 pub bootstrap_samples: usize,
37 pub random_seed: Option<u64>,
39 pub numerical_precision: f64,
41}
42
43#[derive(Debug, Clone, Copy)]
44pub enum MissingValueStrategy {
45 ListwiseDeletion,
47 MeanImputation,
49 MedianImputation,
51 ModeImputation,
53 ForwardFill,
55 BackwardFill,
57}
58
59impl Default for StatsConfig {
60 fn default() -> Self {
61 Self {
62 confidence_level: 0.95,
63 missing_value_strategy: MissingValueStrategy::ListwiseDeletion,
64 bootstrap_samples: 1000,
65 random_seed: None,
66 numerical_precision: 1e-10,
67 }
68 }
69}
70
71impl SciRS2StatsProcessor {
72 pub fn new(config: StatsConfig) -> Self {
74 Self { config }
75 }
76
77 pub fn default() -> Self {
79 Self::new(StatsConfig::default())
80 }
81
82 pub fn config(&self) -> &StatsConfig {
84 &self.config
85 }
86
87 pub fn describe<T: FloatElement>(&self, tensor: &Tensor<T>) -> Result<DescriptiveStats<T>> {
91 let data = self.tensor_to_array1(tensor)?;
92
93 let n = data.len() as f64;
97 let mean = data
98 .iter()
99 .map(|&x| ToPrimitive::to_f64(&x).expect("f64 conversion should succeed"))
100 .sum::<f64>()
101 / n;
102
103 let variance = data
104 .iter()
105 .map(|&x| {
106 let diff = ToPrimitive::to_f64(&x).expect("f64 conversion should succeed") - mean;
107 diff * diff
108 })
109 .sum::<f64>()
110 / (n - 1.0);
111
112 let std_dev = variance.sqrt();
113
114 let skewness = if std_dev > 0.0 {
116 data.iter()
117 .map(|&x| {
118 let z = (ToPrimitive::to_f64(&x).expect("f64 conversion should succeed")
119 - mean)
120 / std_dev;
121 z * z * z
122 })
123 .sum::<f64>()
124 / n
125 } else {
126 0.0
127 };
128
129 let kurtosis = if std_dev > 0.0 {
131 data.iter()
132 .map(|&x| {
133 let z = (ToPrimitive::to_f64(&x).expect("f64 conversion should succeed")
134 - mean)
135 / std_dev;
136 z * z * z * z
137 })
138 .sum::<f64>()
139 / n
140 - 3.0 } else {
142 0.0
143 };
144
145 let mut sorted_data: Vec<f64> = data
147 .iter()
148 .map(|&x| ToPrimitive::to_f64(&x).expect("f64 conversion should succeed"))
149 .collect();
150 sorted_data.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
151
152 let q25 = self.percentile(&sorted_data, 25.0);
153 let median = self.percentile(&sorted_data, 50.0);
154 let q75 = self.percentile(&sorted_data, 75.0);
155
156 Ok(DescriptiveStats {
157 count: n as usize,
158 mean: T::from_f64(mean).expect("f64 conversion should succeed"),
159 std_dev: T::from_f64(std_dev).expect("f64 conversion should succeed"),
160 variance: T::from_f64(variance).expect("f64 conversion should succeed"),
161 skewness: T::from_f64(skewness).expect("f64 conversion should succeed"),
162 kurtosis: T::from_f64(kurtosis).expect("f64 conversion should succeed"),
163 min: T::from_f64(sorted_data[0]).expect("f64 conversion should succeed"),
164 max: T::from_f64(sorted_data[sorted_data.len() - 1])
165 .expect("f64 conversion should succeed"),
166 q25: T::from_f64(q25).expect("f64 conversion should succeed"),
167 median: T::from_f64(median).expect("f64 conversion should succeed"),
168 q75: T::from_f64(q75).expect("f64 conversion should succeed"),
169 iqr: T::from_f64(q75 - q25).expect("f64 conversion should succeed"),
170 })
171 }
172
173 pub fn correlation_matrix<T: FloatElement>(
175 &self,
176 tensor: &Tensor<T>,
177 ) -> Result<CorrelationResult<T>> {
178 let data = self.tensor_to_array2(tensor)?;
179
180 let (n_rows, n_cols) = data.dim();
182 let mut correlation_matrix = Array2::zeros((n_cols, n_cols));
183
184 for i in 0..n_cols {
186 for j in 0..n_cols {
187 let col_i: Vec<f64> = (0..n_rows)
188 .map(|row| {
189 ToPrimitive::to_f64(&data[[row, i]]).expect("f64 conversion should succeed")
190 })
191 .collect();
192 let col_j: Vec<f64> = (0..n_rows)
193 .map(|row| {
194 ToPrimitive::to_f64(&data[[row, j]]).expect("f64 conversion should succeed")
195 })
196 .collect();
197
198 let correlation = self.pearson_correlation(&col_i, &col_j);
199 correlation_matrix[[i, j]] = correlation;
200 }
201 }
202
203 let corr_tensor = self.array2_to_tensor(&correlation_matrix)?;
205
206 Ok(CorrelationResult {
207 correlation_matrix: corr_tensor,
208 method: CorrelationMethod::Pearson,
209 significant_pairs: Vec::new(), })
211 }
212
213 pub fn one_sample_ttest<T: FloatElement>(
217 &self,
218 data: &Tensor<T>,
219 expected_mean: T,
220 ) -> Result<TTestResult<T>> {
221 let values = self.tensor_to_array1(data)?;
222 let n = values.len() as f64;
223 let expected = ToPrimitive::to_f64(&expected_mean).expect("f64 conversion should succeed");
224
225 let sample_mean = values
227 .iter()
228 .map(|&x| ToPrimitive::to_f64(&x).expect("f64 conversion should succeed"))
229 .sum::<f64>()
230 / n;
231
232 let sample_var = values
233 .iter()
234 .map(|&x| {
235 let diff =
236 ToPrimitive::to_f64(&x).expect("f64 conversion should succeed") - sample_mean;
237 diff * diff
238 })
239 .sum::<f64>()
240 / (n - 1.0);
241
242 let standard_error = (sample_var / n).sqrt();
243 let t_statistic = (sample_mean - expected) / standard_error;
244 let degrees_of_freedom = n - 1.0;
245
246 let p_value = 2.0 * (1.0 - self.t_cdf(t_statistic.abs(), degrees_of_freedom));
248
249 Ok(TTestResult {
250 t_statistic: T::from_f64(t_statistic).expect("f64 conversion should succeed"),
251 p_value: T::from_f64(p_value).expect("f64 conversion should succeed"),
252 degrees_of_freedom: T::from_f64(degrees_of_freedom)
253 .expect("f64 conversion should succeed"),
254 confidence_interval: self.compute_confidence_interval(
255 sample_mean,
256 standard_error,
257 degrees_of_freedom,
258 ),
259 effect_size: T::from_f64((sample_mean - expected) / sample_var.sqrt())
260 .expect("f64 conversion should succeed"),
261 })
262 }
263
264 pub fn two_sample_ttest<T: FloatElement>(
266 &self,
267 group1: &Tensor<T>,
268 group2: &Tensor<T>,
269 equal_variance: bool,
270 ) -> Result<TTestResult<T>> {
271 let data1 = self.tensor_to_array1(group1)?;
272 let data2 = self.tensor_to_array1(group2)?;
273
274 let n1 = data1.len() as f64;
276 let n2 = data2.len() as f64;
277
278 let mean1 = data1
279 .iter()
280 .map(|&x| ToPrimitive::to_f64(&x).expect("f64 conversion should succeed"))
281 .sum::<f64>()
282 / n1;
283 let mean2 = data2
284 .iter()
285 .map(|&x| ToPrimitive::to_f64(&x).expect("f64 conversion should succeed"))
286 .sum::<f64>()
287 / n2;
288
289 let var1 = data1
290 .iter()
291 .map(|&x| {
292 let diff = ToPrimitive::to_f64(&x).expect("f64 conversion should succeed") - mean1;
293 diff * diff
294 })
295 .sum::<f64>()
296 / (n1 - 1.0);
297
298 let var2 = data2
299 .iter()
300 .map(|&x| {
301 let diff = ToPrimitive::to_f64(&x).expect("f64 conversion should succeed") - mean2;
302 diff * diff
303 })
304 .sum::<f64>()
305 / (n2 - 1.0);
306
307 let (t_statistic, degrees_of_freedom, standard_error) = if equal_variance {
308 let pooled_var = ((n1 - 1.0) * var1 + (n2 - 1.0) * var2) / (n1 + n2 - 2.0);
310 let se = (pooled_var * (1.0 / n1 + 1.0 / n2)).sqrt();
311 let t = (mean1 - mean2) / se;
312 let df = n1 + n2 - 2.0;
313 (t, df, se)
314 } else {
315 let se = (var1 / n1 + var2 / n2).sqrt();
317 let t = (mean1 - mean2) / se;
318 let df = (var1 / n1 + var2 / n2).powi(2)
319 / ((var1 / n1).powi(2) / (n1 - 1.0) + (var2 / n2).powi(2) / (n2 - 1.0));
320 (t, df, se)
321 };
322
323 let p_value = 2.0 * (1.0 - self.t_cdf(t_statistic.abs(), degrees_of_freedom));
324
325 Ok(TTestResult {
326 t_statistic: T::from_f64(t_statistic).expect("f64 conversion should succeed"),
327 p_value: T::from_f64(p_value).expect("f64 conversion should succeed"),
328 degrees_of_freedom: T::from_f64(degrees_of_freedom)
329 .expect("f64 conversion should succeed"),
330 confidence_interval: self.compute_confidence_interval(
331 mean1 - mean2,
332 standard_error,
333 degrees_of_freedom,
334 ),
335 effect_size: T::from_f64((mean1 - mean2) / ((var1 + var2) / 2.0).sqrt())
336 .expect("f64 conversion should succeed"), })
338 }
339
340 pub fn linear_regression<T: FloatElement>(
344 &self,
345 x: &Tensor<T>,
346 y: &Tensor<T>,
347 ) -> Result<RegressionResult<T>> {
348 let x_data = self.tensor_to_array1(x)?;
349 let y_data = self.tensor_to_array1(y)?;
350
351 if x_data.len() != y_data.len() {
352 return Err(TorshError::RuntimeError(
353 "X and Y must have same length".to_string(),
354 ));
355 }
356
357 let n = x_data.len() as f64;
359
360 let x_vals: Vec<f64> = x_data
361 .iter()
362 .map(|&v| ToPrimitive::to_f64(&v).expect("f64 conversion should succeed"))
363 .collect();
364 let y_vals: Vec<f64> = y_data
365 .iter()
366 .map(|&v| ToPrimitive::to_f64(&v).expect("f64 conversion should succeed"))
367 .collect();
368
369 let x_mean = x_vals.iter().sum::<f64>() / n;
370 let y_mean = y_vals.iter().sum::<f64>() / n;
371
372 let numerator: f64 = x_vals
373 .iter()
374 .zip(y_vals.iter())
375 .map(|(&x, &y)| (x - x_mean) * (y - y_mean))
376 .sum();
377
378 let denominator: f64 = x_vals.iter().map(|&x| (x - x_mean) * (x - x_mean)).sum();
379
380 let slope = numerator / denominator;
381 let intercept = y_mean - slope * x_mean;
382
383 let y_pred: Vec<f64> = x_vals.iter().map(|&x| intercept + slope * x).collect();
385 let ss_res: f64 = y_vals
386 .iter()
387 .zip(y_pred.iter())
388 .map(|(&y, &pred)| (y - pred) * (y - pred))
389 .sum();
390
391 let ss_tot: f64 = y_vals.iter().map(|&y| (y - y_mean) * (y - y_mean)).sum();
392
393 let r_squared = 1.0 - ss_res / ss_tot;
394
395 let mse = ss_res / (n - 2.0);
397 let slope_se = (mse / denominator).sqrt();
398 let intercept_se = (mse * (1.0 / n + x_mean * x_mean / denominator)).sqrt();
399
400 Ok(RegressionResult {
401 coefficients: vec![
402 T::from_f64(intercept).expect("f64 conversion should succeed"),
403 T::from_f64(slope).expect("f64 conversion should succeed"),
404 ],
405 standard_errors: vec![
406 T::from_f64(intercept_se).expect("f64 conversion should succeed"),
407 T::from_f64(slope_se).expect("f64 conversion should succeed"),
408 ],
409 r_squared: T::from_f64(r_squared).expect("f64 conversion should succeed"),
410 adjusted_r_squared: T::from_f64(1.0 - (1.0 - r_squared) * (n - 1.0) / (n - 2.0))
411 .expect("f64 conversion should succeed"),
412 f_statistic: T::from_f64(r_squared * (n - 2.0) / (1.0 - r_squared))
413 .expect("f64 conversion should succeed"),
414 residuals: {
415 let residuals_data: Vec<T> = y_vals
416 .iter()
417 .zip(y_pred.iter())
418 .map(|(&y, &pred)| {
419 T::from_f64(y - pred).expect("f64 conversion should succeed")
420 })
421 .collect();
422 let len = residuals_data.len();
423 Tensor::from_vec(residuals_data, &[len])?
424 },
425 })
426 }
427
428 pub fn fit_normal_distribution<T: FloatElement>(
432 &self,
433 data: &Tensor<T>,
434 ) -> Result<DistributionFit<T>> {
435 let values = self.tensor_to_array1(data)?;
436
437 let n = values.len() as f64;
439 let mean = values
440 .iter()
441 .map(|&x| ToPrimitive::to_f64(&x).expect("f64 conversion should succeed"))
442 .sum::<f64>()
443 / n;
444 let variance = values
445 .iter()
446 .map(|&x| {
447 let diff = ToPrimitive::to_f64(&x).expect("f64 conversion should succeed") - mean;
448 diff * diff
449 })
450 .sum::<f64>()
451 / (n - 1.0);
452
453 let std_dev = variance.sqrt();
454
455 let log_likelihood = values
457 .iter()
458 .map(|&x| {
459 let z = (ToPrimitive::to_f64(&x).expect("f64 conversion should succeed") - mean)
460 / std_dev;
461 -0.5 * (2.0 * std::f64::consts::PI).ln() - std_dev.ln() - 0.5 * z * z
462 })
463 .sum::<f64>();
464
465 Ok(DistributionFit {
466 distribution_type: DistributionType::Normal,
467 parameters: HashMap::from([
468 (
469 "mean".to_string(),
470 T::from_f64(mean).expect("f64 conversion should succeed"),
471 ),
472 (
473 "std_dev".to_string(),
474 T::from_f64(std_dev).expect("f64 conversion should succeed"),
475 ),
476 ]),
477 log_likelihood: T::from_f64(log_likelihood).expect("f64 conversion should succeed"),
478 aic: T::from_f64(-2.0 * log_likelihood + 2.0 * 2.0)
479 .expect("f64 conversion should succeed"), bic: T::from_f64(-2.0 * log_likelihood + 2.0 * n.ln())
481 .expect("f64 conversion should succeed"),
482 goodness_of_fit: {
483 let values_f64: Vec<f64> = values
484 .iter()
485 .map(|&x| ToPrimitive::to_f64(&x).expect("f64 conversion should succeed"))
486 .collect();
487 self.kolmogorov_smirnov_test(&values_f64, mean, std_dev)
488 },
489 })
490 }
491
492 fn tensor_to_array1<T: TensorElement>(&self, tensor: &Tensor<T>) -> Result<Array1<T>> {
495 let data = tensor.to_vec()?;
496 let shape = tensor.shape();
497 if shape.dims().len() != 1 {
498 return Err(TorshError::RuntimeError("Expected 1D tensor".to_string()));
499 }
500
501 Array1::from_vec(data)
502 .into_shape_with_order((shape.dims()[0],))
503 .map_err(|e| TorshError::RuntimeError(format!("Array conversion failed: {}", e)))
504 }
505
506 fn tensor_to_array2<T: TensorElement>(&self, tensor: &Tensor<T>) -> Result<Array2<T>> {
507 let data = tensor.to_vec()?;
508 let shape = tensor.shape();
509 if shape.dims().len() != 2 {
510 return Err(TorshError::RuntimeError("Expected 2D tensor".to_string()));
511 }
512
513 Array2::from_shape_vec((shape.dims()[0], shape.dims()[1]), data)
514 .map_err(|e| TorshError::RuntimeError(format!("Array conversion failed: {}", e)))
515 }
516
517 fn array2_to_tensor<T: TensorElement>(&self, array: &Array2<f64>) -> Result<Tensor<T>> {
518 let data: Vec<T> = array
519 .iter()
520 .map(|&x| T::from_f64(x).expect("f64 conversion should succeed"))
521 .collect();
522 let shape = vec![array.nrows(), array.ncols()];
523 Tensor::from_vec(data, &shape)
524 }
525
526 fn percentile(&self, sorted_data: &[f64], percentile: f64) -> f64 {
527 let n = sorted_data.len();
528 let index = (percentile / 100.0) * ((n - 1) as f64);
529 let lower = index.floor() as usize;
530 let upper = index.ceil() as usize;
531
532 if lower == upper {
533 sorted_data[lower]
534 } else {
535 let weight = index - (lower as f64);
536 sorted_data[lower] * (1.0 - weight) + sorted_data[upper] * weight
537 }
538 }
539
540 fn pearson_correlation(&self, x: &[f64], y: &[f64]) -> f64 {
541 let n = x.len() as f64;
542 let x_mean = x.iter().sum::<f64>() / n;
543 let y_mean = y.iter().sum::<f64>() / n;
544
545 let numerator: f64 = x
546 .iter()
547 .zip(y.iter())
548 .map(|(&xi, &yi)| (xi - x_mean) * (yi - y_mean))
549 .sum();
550
551 let x_var: f64 = x.iter().map(|&xi| (xi - x_mean) * (xi - x_mean)).sum();
552 let y_var: f64 = y.iter().map(|&yi| (yi - y_mean) * (yi - y_mean)).sum();
553
554 numerator / (x_var * y_var).sqrt()
555 }
556
557 fn t_cdf(&self, t: f64, df: f64) -> f64 {
558 0.5 + 0.5 * (t / (1.0 + t * t / df).sqrt()).tanh()
561 }
562
563 fn compute_confidence_interval<T: FloatElement>(
564 &self,
565 estimate: f64,
566 standard_error: f64,
567 degrees_of_freedom: f64,
568 ) -> (T, T) {
569 let alpha = 1.0 - self.config.confidence_level;
571
572 let t_critical = if degrees_of_freedom > 30.0 {
575 if alpha < 0.02 {
578 2.576 } else {
580 1.96 }
582 } else {
583 let base_t = if alpha < 0.02 { 2.8 } else { 2.1 };
586 base_t * (1.0 + 5.0 / degrees_of_freedom).sqrt()
587 };
588
589 let _ = (alpha, degrees_of_freedom, t_critical); let margin_of_error = t_critical * standard_error;
593 (
594 T::from_f64(estimate - margin_of_error).expect("f64 conversion should succeed"),
595 T::from_f64(estimate + margin_of_error).expect("f64 conversion should succeed"),
596 )
597 }
598
599 fn kolmogorov_smirnov_test(&self, data: &[f64], mean: f64, std_dev: f64) -> f64 {
600 if data.is_empty() || std_dev <= 0.0 {
602 return 1.0; }
604
605 let n = data.len() as f64;
606 let mut sorted_data = data.to_vec();
607 sorted_data.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
608
609 let mut max_deviation = 0.0f64;
611
612 for (i, &x) in sorted_data.iter().enumerate() {
613 let empirical_cdf = (i + 1) as f64 / n;
615
616 let z = (x - mean) / std_dev;
618 let theoretical_cdf = 0.5 * (1.0 + libm::erf(z / std::f64::consts::SQRT_2));
620
621 let deviation = (empirical_cdf - theoretical_cdf).abs();
623 max_deviation = max_deviation.max(deviation);
624 }
625
626 let ks_statistic = max_deviation * n.sqrt();
629
630 let p_value = (-2.0 * ks_statistic * ks_statistic).exp();
633 1.0 - p_value
634 }
635}
636
637#[derive(Debug, Clone)]
639pub struct DescriptiveStats<T: TensorElement> {
640 pub count: usize,
641 pub mean: T,
642 pub std_dev: T,
643 pub variance: T,
644 pub skewness: T,
645 pub kurtosis: T,
646 pub min: T,
647 pub max: T,
648 pub q25: T,
649 pub median: T,
650 pub q75: T,
651 pub iqr: T,
652}
653
654#[derive(Debug, Clone)]
656pub struct CorrelationResult<T: TensorElement> {
657 pub correlation_matrix: Tensor<T>,
658 pub method: CorrelationMethod,
659 pub significant_pairs: Vec<(usize, usize, T)>, }
661
662#[derive(Debug, Clone, Copy)]
663pub enum CorrelationMethod {
664 Pearson,
665 Spearman,
666 Kendall,
667}
668
669#[derive(Debug, Clone)]
671pub struct TTestResult<T: TensorElement> {
672 pub t_statistic: T,
673 pub p_value: T,
674 pub degrees_of_freedom: T,
675 pub confidence_interval: (T, T),
676 pub effect_size: T,
677}
678
679#[derive(Debug, Clone)]
681pub struct RegressionResult<T: TensorElement> {
682 pub coefficients: Vec<T>,
683 pub standard_errors: Vec<T>,
684 pub r_squared: T,
685 pub adjusted_r_squared: T,
686 pub f_statistic: T,
687 pub residuals: Tensor<T>,
688}
689
690#[derive(Debug, Clone)]
692pub struct DistributionFit<T: TensorElement> {
693 pub distribution_type: DistributionType,
694 pub parameters: HashMap<String, T>,
695 pub log_likelihood: T,
696 pub aic: T,
697 pub bic: T,
698 pub goodness_of_fit: f64,
699}
700
701#[derive(Debug, Clone, Copy)]
702pub enum DistributionType {
703 Normal,
704 Exponential,
705 Gamma,
706 Beta,
707 Poisson,
708 Binomial,
709 Uniform,
710}
711
712pub use crate::stats::{HistogramConfig, StatMode};