Struct PendulumEnvironment 

pub struct PendulumEnvironment { /* private fields */ }

Pendulum environment for continuous control

Implementations

impl PendulumEnvironment

pub fn new() -> Self

Create new pendulum environment

Examples found in repository

examples/continuous_rl.rs (line 37)

fn test_pendulum_dynamics() -> Result<()> {
    let mut env = PendulumEnvironment::new();

    println!("   Initial state: {:?}", env.state());
    println!("   Action bounds: {:?}", env.action_bounds());

    // Run a few steps with different actions
    let actions = vec![
        Array1::from_vec(vec![0.0]),  // No torque
        Array1::from_vec(vec![2.0]),  // Max positive torque
        Array1::from_vec(vec![-2.0]), // Max negative torque
    ];

    for (i, action) in actions.iter().enumerate() {
        let state = env.reset();
        let (next_state, reward, done) = env.step(action.clone())?;

        println!("\n   Step {} with action {:.1}:", i + 1, action[0]);
        println!(
            "     State: [θ_cos={:.3}, θ_sin={:.3}, θ_dot={:.3}]",
            state[0], state[1], state[2]
        );
        println!(
            "     Next: [θ_cos={:.3}, θ_sin={:.3}, θ_dot={:.3}]",
            next_state[0], next_state[1], next_state[2]
        );
        println!("     Reward: {reward:.3}, Done: {done}");
    }

    Ok(())
}

/// Train QDDPG on pendulum control
fn train_qddpg_pendulum() -> Result<()> {
    let state_dim = 3;
    let action_dim = 1;
    let action_bounds = vec![(-2.0, 2.0)];
    let num_qubits = 4;
    let buffer_capacity = 10000;

    // Create QDDPG agent
    let mut agent = QuantumDDPG::new(
        state_dim,
        action_dim,
        action_bounds,
        num_qubits,
        buffer_capacity,
    )?;

    // Create environment
    let mut env = PendulumEnvironment::new();

    // Create optimizers
    let mut actor_optimizer = Adam::new(0.001);
    let mut critic_optimizer = Adam::new(0.001);

    // Train for a few episodes (reduced for demo)
    let episodes = 50;
    println!("   Training QDDPG for {episodes} episodes...");

    let rewards = agent.train(
        &mut env,
        episodes,
        &mut actor_optimizer,
        &mut critic_optimizer,
    )?;

    // Print training statistics
    let avg_initial = rewards[..10].iter().sum::<f64>() / 10.0;
    let avg_final = rewards[rewards.len() - 10..].iter().sum::<f64>() / 10.0;

    println!("\n   Training Statistics:");
    println!("   - Average initial reward: {avg_initial:.2}");
    println!("   - Average final reward: {avg_final:.2}");
    println!("   - Improvement: {:.2}", avg_final - avg_initial);

    // Test trained agent
    println!("\n   Testing trained agent...");
    test_trained_agent(&agent, &mut env)?;

    Ok(())
}

/// Test a trained agent
fn test_trained_agent(agent: &QuantumDDPG, env: &mut dyn ContinuousEnvironment) -> Result<()> {
    let test_episodes = 5;
    let mut test_rewards = Vec::new();

    for episode in 0..test_episodes {
        let mut state = env.reset();
        let mut episode_reward = 0.0;
        let mut done = false;
        let mut steps = 0;

        while !done && steps < 200 {
            let action = agent.get_action(&state, false)?; // No exploration
            let (next_state, reward, is_done) = env.step(action.clone())?;

            state = next_state;
            episode_reward += reward;
            done = is_done;
            steps += 1;
        }

        test_rewards.push(episode_reward);
        println!(
            "   Test episode {}: Reward = {:.2}, Steps = {}",
            episode + 1,
            episode_reward,
            steps
        );
    }

    let avg_test = test_rewards.iter().sum::<f64>() / f64::from(test_episodes);
    println!("   Average test reward: {avg_test:.2}");

    Ok(())
}

/// Compare trained policy with random policy
fn compare_policies() -> Result<()> {
    let mut env = PendulumEnvironment::new();
    let episodes = 10;

    // Random policy performance
    println!("   Random Policy Performance:");
    let mut random_rewards = Vec::new();

    for _ in 0..episodes {
        let mut state = env.reset();
        let mut episode_reward = 0.0;
        let mut done = false;

        while !done {
            // Random action in bounds
            let action = Array1::from_vec(vec![4.0f64.mul_add(thread_rng().gen::<f64>(), -2.0)]);

            let (next_state, reward, is_done) = env.step(action)?;
            state = next_state;
            episode_reward += reward;
            done = is_done;
        }

        random_rewards.push(episode_reward);
    }

    let avg_random = random_rewards.iter().sum::<f64>() / f64::from(episodes);
    println!("   Average random policy reward: {avg_random:.2}");

    // Simple control policy (proportional control)
    println!("\n   Simple Control Policy Performance:");
    let mut control_rewards = Vec::new();

    for _ in 0..episodes {
        let mut state = env.reset();
        let mut episode_reward = 0.0;
        let mut done = false;

        while !done {
            // Proportional control: torque = -k * theta
            let theta = state[1].atan2(state[0]); // Reconstruct angle
            let action = Array1::from_vec(vec![(-2.0 * theta).clamp(-2.0, 2.0)]);

            let (next_state, reward, is_done) = env.step(action)?;
            state = next_state;
            episode_reward += reward;
            done = is_done;
        }

        control_rewards.push(episode_reward);
    }

    let avg_control = control_rewards.iter().sum::<f64>() / f64::from(episodes);
    println!("   Average control policy reward: {avg_control:.2}");

    println!("\n   Performance Summary:");
    println!("   - Random policy: {avg_random:.2}");
    println!("   - Simple control: {avg_control:.2}");
    println!("   - Improvement: {:.2}", avg_control - avg_random);

    Ok(())
}

Trait Implementations

impl ContinuousEnvironment for PendulumEnvironment

fn state(&self) -> Array1<f64>

Gets the current state

fn action_bounds(&self) -> Vec<(f64, f64)>

Gets the action space bounds (min, max) for each dimension

fn step(&mut self, action: Array1<f64>) -> Result<(Array1<f64>, f64, bool)>

Takes a continuous action and returns reward and next state

fn reset(&mut self) -> Array1<f64>

Resets the environment

fn state_dim(&self) -> usize

Get state dimension

fn action_dim(&self) -> usize

Get action dimension