thrust_rl/env/games/
matching_pennies.rs

1//! Matching Pennies — 2-agent zero-sum smoke env.
2//!
3//! Canonical mixed-strategy testbed for multi-agent RL: agent 0 (the
4//! "matcher") wins +1 when both actions match; agent 1 (the
5//! "mismatcher") wins +1 when they differ. The unique Nash equilibrium
6//! is the uniform distribution `(0.5, 0.5)` for both agents, which makes
7//! this env the standard smoke test for any mixed-equilibrium learner
8//! (PSRO, NFSP, fictitious play).
9//!
10//! # Why a fresh env instead of reusing `SimpleBandit`?
11//!
12//! `SimpleBandit` is a single-agent contextual bandit with `Environment`
13//! semantics. The PSRO trainer ingests a [`JointEnv`] (see
14//! [`crate::multi_agent::joint`]) where every agent receives the same
15//! observation and a synchronized step result with per-agent rewards.
16//! Matching pennies is the smallest 2-agent zero-sum env that fits the
17//! [`JointEnv`] surface, so we implement it directly here without
18//! touching the heavier `Environment` / `MultiAgentEnvironment` traits.
19//!
20//! # Observation
21//!
22//! Observation is a constant scalar `[0.0]` (single-dim). Both agents
23//! see the same observation on every step — this matches the
24//! "globally-shared observation" constraint inherited from
25//! [`crate::multi_agent::joint::JointMultiAgentTrainer`]. Matching
26//! pennies is a *stateless* simultaneous-move game; the observation is
27//! present only to satisfy the trainer's interface, not because the
28//! agents need state.
29//!
30//! # Action / reward
31//!
32//! - Action: `0` or `1`, single discrete dim per agent.
33//! - Reward: `+1` to the matcher (agent 0) and `-1` to the mismatcher (agent 1)
34//!   when actions match, and the inverse when they differ. The game is zero-sum
35//!   every step.
36//!
37//! # Episode length
38//!
39//! Each `reset_joint` starts a fresh episode of length
40//! [`MatchingPennies::EPISODE_LEN`] steps; `step_joint` sets
41//! `done = true` on the final step. The trainer wraps this with its own
42//! rollout-length loop, so the episode length only affects how
43//! frequently the `done` flag fires; the optimal mixed strategy is the
44//! same regardless of episode length.
45
46use crate::multi_agent::joint::{JointEnv, JointStepResult};
47
48/// 2-agent zero-sum matching-pennies env.
49///
50/// Agent 0 ("matcher"): reward `+1` when both actions agree, `-1` otherwise.
51/// Agent 1 ("mismatcher"): the negation.
52#[derive(Debug, Clone)]
53pub struct MatchingPennies {
54    /// Number of steps elapsed in the current episode.
55    step: usize,
56}
57
58impl MatchingPennies {
59    /// Episode length used by `step_joint` to fire the `done` flag.
60    pub const EPISODE_LEN: usize = 16;
61
62    /// Number of agents in this env (always 2).
63    pub const NUM_AGENTS: usize = 2;
64
65    /// Observation dimensionality (always 1 — a constant scalar).
66    pub const OBS_DIM: usize = 1;
67
68    /// Per-agent action cardinality (always 2 — heads/tails).
69    pub const ACTION_DIM: usize = 2;
70
71    /// Construct a fresh env.
72    pub fn new() -> Self {
73        Self { step: 0 }
74    }
75}
76
77impl Default for MatchingPennies {
78    fn default() -> Self {
79        Self::new()
80    }
81}
82
83impl JointEnv for MatchingPennies {
84    fn reset_joint(&mut self, _seed: Option<u64>) -> Vec<Vec<f32>> {
85        self.step = 0;
86        // Two agents, each gets the same single-dim constant observation.
87        vec![vec![0.0; Self::OBS_DIM]; Self::NUM_AGENTS]
88    }
89
90    fn step_joint(&mut self, actions: &[Vec<i64>]) -> JointStepResult {
91        debug_assert_eq!(actions.len(), Self::NUM_AGENTS, "matching pennies requires 2 agents");
92        debug_assert_eq!(actions[0].len(), 1, "agent 0 must supply a 1-d action");
93        debug_assert_eq!(actions[1].len(), 1, "agent 1 must supply a 1-d action");
94
95        // Matcher wins (+1) when actions agree; mismatcher wins (+1) when
96        // they disagree. Always zero-sum: rewards sum to zero every step.
97        let a0 = actions[0][0];
98        let a1 = actions[1][0];
99        let matcher_reward = if a0 == a1 { 1.0_f32 } else { -1.0_f32 };
100        let rewards = vec![matcher_reward, -matcher_reward];
101
102        self.step += 1;
103        let done = self.step >= Self::EPISODE_LEN;
104
105        JointStepResult {
106            rewards,
107            done,
108            observations: vec![vec![0.0; Self::OBS_DIM]; Self::NUM_AGENTS],
109        }
110    }
111}
112
113#[cfg(test)]
114mod tests {
115    use super::*;
116
117    #[test]
118    fn test_zero_sum_every_step() {
119        // Reward symmetry: matcher_reward + mismatcher_reward == 0 for
120        // every combination of (a0, a1).
121        for a0 in 0..2 {
122            for a1 in 0..2 {
123                let mut env = MatchingPennies::new();
124                env.reset_joint(None);
125                let res = env.step_joint(&[vec![a0], vec![a1]]);
126                let total: f32 = res.rewards.iter().sum();
127                assert!(
128                    total.abs() < 1e-6,
129                    "rewards not zero-sum for ({a0}, {a1}): {:?}",
130                    res.rewards
131                );
132            }
133        }
134    }
135
136    #[test]
137    fn test_matcher_wins_on_agreement() {
138        for a in 0..2 {
139            let mut env = MatchingPennies::new();
140            env.reset_joint(None);
141            let res = env.step_joint(&[vec![a], vec![a]]);
142            assert_eq!(res.rewards[0], 1.0, "matcher should win on agreement (action {a})");
143            assert_eq!(res.rewards[1], -1.0, "mismatcher should lose on agreement (action {a})");
144        }
145    }
146
147    #[test]
148    fn test_mismatcher_wins_on_disagreement() {
149        let pairs = [(0_i64, 1_i64), (1, 0)];
150        for (a0, a1) in pairs {
151            let mut env = MatchingPennies::new();
152            env.reset_joint(None);
153            let res = env.step_joint(&[vec![a0], vec![a1]]);
154            assert_eq!(res.rewards[0], -1.0, "matcher should lose on ({a0}, {a1})");
155            assert_eq!(res.rewards[1], 1.0, "mismatcher should win on ({a0}, {a1})");
156        }
157    }
158
159    #[test]
160    fn test_episode_done_after_episode_len() {
161        let mut env = MatchingPennies::new();
162        env.reset_joint(None);
163        for step in 0..MatchingPennies::EPISODE_LEN {
164            let res = env.step_joint(&[vec![0], vec![0]]);
165            let expected_done = step + 1 >= MatchingPennies::EPISODE_LEN;
166            assert_eq!(res.done, expected_done, "done flag wrong at step {step}");
167        }
168    }
169
170    #[test]
171    fn test_reset_restores_step_counter() {
172        let mut env = MatchingPennies::new();
173        env.reset_joint(None);
174        for _ in 0..MatchingPennies::EPISODE_LEN {
175            env.step_joint(&[vec![0], vec![0]]);
176        }
177        // Reset and verify we get EPISODE_LEN more steps before done.
178        env.reset_joint(None);
179        for step in 0..MatchingPennies::EPISODE_LEN - 1 {
180            let res = env.step_joint(&[vec![0], vec![0]]);
181            assert!(!res.done, "should not be done at step {step} after reset");
182        }
183        let last = env.step_joint(&[vec![0], vec![0]]);
184        assert!(last.done, "should be done on final step after reset");
185    }
186
187    #[test]
188    fn test_observation_shape() {
189        let mut env = MatchingPennies::new();
190        let obs = env.reset_joint(None);
191        assert_eq!(obs.len(), MatchingPennies::NUM_AGENTS);
192        for o in &obs {
193            assert_eq!(o.len(), MatchingPennies::OBS_DIM);
194        }
195        let res = env.step_joint(&[vec![0], vec![0]]);
196        assert_eq!(res.observations.len(), MatchingPennies::NUM_AGENTS);
197        for o in &res.observations {
198            assert_eq!(o.len(), MatchingPennies::OBS_DIM);
199        }
200    }
201}
thrust_rl/env/games/matching_pennies.rs

thrust_rl/env/games/
matching_pennies.rs
