optirs-core 0.3.2

OptiRS core optimization algorithms and utilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

//! Example demonstrating gradient clipping techniques

use optirs_core::{
    // use statrs::statistics::Statistics; // statrs not available
    gradient_processing::GradientProcessor,
    optimizers::{Adam, Optimizer},
};
use scirs2_core::ndarray::Array1;

#[allow(dead_code)]
fn main() {
    // Create example parameters and large gradients (simulating gradient explosion)
    let params = Array1::zeros(5);
    let mut large_gradients = Array1::from_vec(vec![100.0, -200.0, 300.0, -150.0, 250.0]);

    println!("Original gradients: {:?}", large_gradients);

    // Create gradient processor with L2 norm clipping
    let mut processor = GradientProcessor::new();

    // Configure L2 norm clipping
    processor.set_max_norm(10.0);

    // Clip gradients
    processor
        .process(&mut large_gradients)
        .expect("unwrap failed");
    println!("Clipped gradients (L2 norm): {:?}", large_gradients);
    println!(
        "L2 norm of clipped gradients: {:.4}",
        large_gradients.iter().map(|x| x * x).sum::<f64>().sqrt()
    );

    // Use value clipping
    let mut value_gradients = Array1::from_vec(vec![100.0, -200.0, 300.0, -150.0, 250.0]);
    let mut value_processor = GradientProcessor::new();
    value_processor.set_max_value(50.0).set_min_value(-50.0);

    value_processor
        .process(&mut value_gradients)
        .expect("unwrap failed");
    println!("\nValue clipped gradients (±50): {:?}", value_gradients);

    // Use L1 norm clipping
    let mut l1_gradients = Array1::from_vec(vec![100.0, -200.0, 300.0, -150.0, 250.0]);
    let mut l1_processor = GradientProcessor::new();
    l1_processor.set_max_l1_norm(100.0);

    l1_processor
        .process(&mut l1_gradients)
        .expect("unwrap failed");
    println!("\nL1 norm clipped gradients: {:?}", l1_gradients);

    // Apply gradient clipping in optimization loop
    let mut optimizer = Adam::new(0.001);
    let mut current_params = params.clone();
    let mut processor = GradientProcessor::new();
    processor.set_max_norm(10.0);

    for epoch in 0..5 {
        let mut gradients = Array1::from_vec(vec![
            10.0 * (epoch as f64 + 1.0),
            -20.0 * (epoch as f64 + 1.0),
            15.0 * (epoch as f64 + 1.0),
            -5.0 * (epoch as f64 + 1.0),
            12.0 * (epoch as f64 + 1.0),
        ]);

        // Clip gradients before optimization
        processor.process(&mut gradients).expect("unwrap failed");

        // Update parameters with clipped gradients
        current_params = optimizer
            .step(&current_params, &gradients)
            .expect("unwrap failed");

        println!(
            "\nEpoch {}: param_norm={:.4}",
            epoch,
            current_params.iter().map(|x| x * x).sum::<f64>().sqrt()
        );
    }

    // Demonstrate gradient zeroing for small gradients
    let mut small_gradients = Array1::from_vec(vec![0.0001, -0.0002, 0.0003, 0.00001, -0.00005]);
    println!("\nSmall gradients: {:?}", small_gradients);

    // Zero gradients smaller than threshold
    let mut zeroing_processor = GradientProcessor::new();
    zeroing_processor.set_zero_threshold(0.0002);

    zeroing_processor
        .process(&mut small_gradients)
        .expect("unwrap failed");
    println!(
        "Gradients after zeroing small values: {:?}",
        small_gradients
    );

    // Demonstrate gradient centralization
    let mut grad_for_centralization = Array1::from_vec(vec![1.0, 2.0, 3.0, 4.0, 5.0]);
    println!(
        "\nGradients before centralization: {:?}",
        grad_for_centralization
    );

    let mut central_processor = GradientProcessor::new();
    central_processor.set_centralization(true);

    central_processor
        .process(&mut grad_for_centralization)
        .expect("unwrap failed");
    println!(
        "Gradients after centralization: {:?}",
        grad_for_centralization
    );
    println!(
        "Mean after centralization: {:.6}",
        grad_for_centralization.mean().expect("unwrap failed")
    );
}