thrust-rl 0.4.0

[package]
name = "thrust-rl"
version = "0.4.0"
edition = "2024"
authors = ["Robb Walters <agent-4@spheresemi.com>"]
license = "MIT OR Apache-2.0"
description = "High-performance reinforcement learning in Rust with the Burn tensor backend"
repository = "https://github.com/rjwalters/thrust"
homepage = "https://rjwalters.github.io/thrust/"
documentation = "https://docs.rs/thrust-rl"
# Keywords: <=5 entries, each <=20 chars, alphanumeric + '-'/'_'.
keywords = ["reinforcement", "machine-learning", "rl", "ppo", "dqn"]
# Categories from the official crates.io list:
# https://crates.io/category_slugs
categories = ["science", "algorithms", "game-development", "simulation"]
readme = "README.md"
# Restrict the published tarball to the files actually needed to build the
# crate. This keeps the tarball small (well under the 10 MB crates.io
# limit) by excluding tracked model checkpoints, the web demo, the
# bucket-brigade submodule, training scripts, optimization result JSON,
# screenshots, and other non-library assets.
exclude = [
    # Trained model checkpoints (tens of MB) -- consumers build their own.
    "*.pt",
    "*.safetensors",
    "*.onnx",
    "models/**",
    # Web demo assets.
    "web/**",
    # Optional Bucket Brigade submodule (only built with the
    # `env-bucket-brigade` feature; consumers can vendor it themselves).
    "envs/**",
    # Bucket Brigade example -- feature is disabled in v0.1.0 (see above).
    "examples/games/bucket_brigade/**",
    # Out-of-band tooling that is not needed to build the library.
    "scripts/**",
    "loom.sh",
    "Makefile",
    "rustfmt.toml",
    "clippy.toml",
    ".loom/**",
    ".claude/**",
    ".github/**",
    # Project planning docs that aren't consumed by `cargo doc`.
    "docs/WASM_ROADMAP.md",
    "docs/MULTI_AGENT_DESIGN.md",
    # Superseded internal docs — kept in-repo for the record, noise on crates.io.
    "docs/archive/**",
    "package.json",
]

[lib]
crate-type = ["cdylib", "rlib"]

[dependencies]
# Burn 0.21 — pure-Rust tensor framework. After phase 5 of the migration
# (#82), Burn is the only tensor backend in the workspace; the old
# `tch`/libtorch path is gone. Mirrors the dependency stance used by sister
# project geode-fem (see ../geode-fem/Cargo.toml workspace block):
# `default-features = false` keeps the dep small; backends (ndarray, wgpu,
# cuda) are layered on as separate features. The default `training` feature
# only pulls in `ndarray` (CPU) + `autodiff`; opt-in features below add
# GPU backends.
burn = { version = "0.21", default-features = false, features = ["std", "ndarray", "autodiff"], optional = true }

# Direct handle on the NdArray backend crate that `burn` already pulls in. The
# `burn` umbrella crate re-exports the BLAS backends (`accelerate`/`openblas`/
# ...) but does NOT re-export the pure-Rust `multi-threads` feature, which is the
# only threaded-matmul option that needs no system BLAS library. We declare the
# crate here (same version, off by default) purely so the `blas-threaded`
# feature can flip `burn-ndarray/multi-threads` on. Cargo unifies features
# across the two dependency paths, so this toggles threading on the *same*
# compiled `NdArray` backend the rest of the code already uses via
# `burn::backend::NdArray` — no source changes required.
burn-ndarray = { version = "0.21", default-features = false, optional = true }

# Async runtime for vectorization (not available in WASM)
tokio = { version = "1", features = ["full", "rt-multi-thread"], optional = true }

# Data parallelism (not available in WASM)
rayon = { version = "1.8", optional = true }

# Zero-copy operations
bytemuck = { version = "1.14", features = ["derive"] }

# Serialization for checkpoints
serde = { version = "1", features = ["derive"] }
serde_json = "1.0"

# Metrics and logging (simplified for WASM)
tracing = { version = "0.1", optional = true }
tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true }

# Random number generation. `rand 0.9` pulls `getrandom 0.3` transitively; the
# wasm32 backend for getrandom is configured in the wasm-only target table near
# the end of this file (see `[target.'cfg(target_arch = "wasm32")'.dependencies]`).
# `small_rng` enables the fast, small-state `SmallRng` used by the Atari
# preprocessing wrapper for reproducible, seed-controlled sticky actions.
rand = { version = "0.9", features = ["small_rng"] }

# Multi-threading and channels (not available in WASM)
crossbeam-channel = { version = "0.5", optional = true }

# Error handling
anyhow = "1.0"
thiserror = "1.0"

# Date/time utilities (optional for training metadata)
chrono = { version = "0.4", optional = true }

# Bucket Brigade environment (Slepian-Wolf MARL experiments). Default = no
# Python features; pure Rust core.
#
# NOTE for v0.1.0 release: `bucket-brigade-core` lives in the
# `envs/bucket-brigade/` git submodule and has not been published to
# crates.io. `cargo publish` refuses to publish any package with a
# path-only dependency (even an optional one). Consequently this dep is
# commented out below and the `env-bucket-brigade` feature is disabled
# for the published crate. To use the Bucket Brigade env locally, depend
# on `thrust-rl` from a git checkout that includes the submodule and
# uncomment this block plus the `env-bucket-brigade` feature definition.
# See `docs/RELEASING.md` for the full procedure and the follow-up plan
# for restoring this feature in v0.2.x.
# bucket-brigade-core = { path = "envs/bucket-brigade/bucket-brigade-core", default-features = false, optional = true }

# WASM support
wasm-bindgen = { version = "0.2", optional = true }
web-sys = { version = "0.3", features = ["console"], optional = true }
js-sys = { version = "0.3", optional = true }
console_error_panic_hook = { version = "0.1", optional = true }

# wasm32-only deps. `getrandom 0.3` (pulled by `rand 0.9`) has no default
# backend on wasm32-unknown-unknown — it needs the `wasm_js` feature here PLUS
# `--cfg getrandom_backend="wasm_js"` (set in .cargo/config.toml). Gated to
# wasm32 so the wasm-bindgen/js-sys deps this feature pulls stay out of native
# and the published-crate dependency graph.
[target.'cfg(target_arch = "wasm32")'.dependencies]
getrandom = { version = "0.3", features = ["wasm_js"] }

[features]
default = ["training"]
# `training` enables the Burn-backed training stack: tensor ops, autodiff,
# the buffer surfaces, the PPO/DQN trainers, and the policy networks.
# Defaults to Burn's pure-Rust NdArray backend so a fresh checkout builds
# without any external system libraries. GPU backends are opt-in via
# additional feature flags below.
training = ["dep:burn", "tokio", "rayon", "tracing", "tracing-subscriber", "crossbeam-channel", "chrono"]
# Threaded matmul for the CPU NdArray backend (OFF by default). Enables
# `burn-ndarray/multi-threads`, which turns on `matrixmultiply/threading` +
# `ndarray/rayon` — both PURE RUST (matrixmultiply pulls in `thread-tree` +
# `num_cpus`). No system BLAS library is required, so this links cleanly on a
# bare Ubuntu 24.04 node (no apt install) as well as macOS. Intended for the
# batched phases: the PPO update (src/multi_agent/joint.rs) and the average-
# policy supervised training (src/multi_agent/nfsp.rs), where matmuls are large
# enough to amortize thread dispatch.
#
# Compose with `training`, e.g. `--features "training,blas-threaded"`. Tune the
# worker count with RAYON_NUM_THREADS / matrixmultiply's pool sizing.
#
# REPRODUCIBILITY CAVEAT: threaded matmul changes the order of floating-point
# reductions, so bit-exact results may differ run-to-run and from the single-
# threaded path. CI and the default build stay on the deterministic pure-Rust
# single-threaded path; only enable this for throughput-oriented runs where
# bit-reproducibility is not required.
blas-threaded = ["dep:burn-ndarray", "burn-ndarray/multi-threads"]
# Burn wgpu backend — cross-platform GPU (Vulkan/Metal/DX12/WebGPU).
# Compose with `training`, e.g. `--features "training,wgpu"`.
wgpu = ["burn?/wgpu"]
# Burn CUDA backend — Linux/NVIDIA only. Compose with `training`.
cuda = ["burn?/cuda"]
# Opt-in mixed-precision (f16) GPU training path (issue #270, epic #267).
# Marker feature: when it is on *and* a GPU backend feature is also on, the
# fp16 example binary (`train_pong_dqn_fp16`) selects `f16` as the backend
# float element instead of `f32`, and enables manual dynamic loss scaling.
#
# It pulls in `burn?/wgpu` so that `--all-features` (the CI `docs` job) always
# has a GPU backend active — otherwise the `compile_error!` guard in the
# example would fire during the docs build. The example itself emits a
# `compile_error!` if `training-fp16` is enabled without `cuda` OR `wgpu`
# (NdArray/CPU has no f16 support: `FloatNdArrayElement` is impl'd only for
# f32/f64). The verified runtime path is CUDA (NVIDIA Ampere+); wgpu/Metal f16
# compiles but is not runtime-verified and bf16 matmul is unavailable there
# (see #305).
#
# Compose as `--features "training,env-atari,cuda,training-fp16"`.
training-fp16 = ["burn?/wgpu"]
# Burn collective ops (burn-collective) — in-process/cross-node all_reduce,
# broadcast, reduce for data-parallel gradient sync. OFF by default and NOT in
# the CI matrix: opt-in only, gated so the default/`training` builds stay lean.
# Compose with `training`, e.g. `--features "training,collective"`. This is the
# feature the distributed-training epic (see docs/DISTRIBUTED_TRAINING_DESIGN.md)
# gates future DDP work on; Phase 1 spike results are recorded in that doc.
collective = ["burn?/collective"]
wasm = ["wasm-bindgen", "web-sys", "js-sys", "console_error_panic_hook"]  # Pure Rust for WASM
# Bucket Brigade env adapter for the Slepian-Wolf MARL experiments. Pulls
# in the bucket-brigade-core crate as a pure-Rust dep --- does *not*
# require the `training` feature on its own. Disabled for v0.1.0 because
# bucket-brigade-core is path-only (see comment in [dependencies]); the
# `bucket_brigade` module in src/ is `#[cfg(feature = "env-bucket-brigade")]`
# gated and will simply be omitted from the published crate. Re-enable
# locally by uncommenting both this line and the dependency above.
# env-bucket-brigade = ["bucket-brigade-core"]
# Atari (ALE) env adapter. Option D (subprocess via ale-py) — see
# docs/ALE_BINDING_STRATEGY.md. No native/path dependency: the emulator runs as
# an external `ale-py` process invoked at runtime, so this is a pure compile-time
# flag with no `[dependencies]` entry and no publish `sed` workaround. Compose
# with `training`, e.g. `--features "training,env-atari"`.
env-atari = []

[dev-dependencies]
criterion = "0.5"
tempfile = "3.8"

[[bench]]
name = "trainer_throughput"
harness = false
required-features = ["training"]

[profile.release]
opt-level = 3
lto = true
codegen-units = 1

[profile.bench]
inherits = "release"

# Bandit examples — Burn-backed SimpleBandit PPO trainer (originally the
# phase-1 scout trainer; promoted to the canonical bandit example in
# phase 5 when the tch sibling was removed).
[[example]]
name = "train_simple_bandit"
path = "examples/games/bandit/train_simple_bandit.rs"
required-features = ["training"]

# Canonical CartPole trainers on the Burn stack. Resurrected from the
# pre-Burn deletion (PR #98) per issue #101. See each example's module
# doc for the algorithm + hyperparameters.
[[example]]
name = "train_cartpole_modern"
path = "examples/games/cartpole/train_cartpole_modern.rs"
required-features = ["training"]

[[example]]
name = "train_cartpole_dqn"
path = "examples/games/cartpole/train_cartpole_dqn.rs"
required-features = ["training"]

# Recurrent PPO on velocity-masked CartPole (Phase 3 of the recurrent-policy
# epic #262): trains an LstmBurnPolicy with RecurrentPPOTrainer on the
# FlickeringCartPole POMDP (4-D obs blanked to zeros with probability p, the
# Hausknecht & Stone 2015 protocol) and contrasts it against a feedforward MLP
# baseline on the same flickering stream — memory is load-bearing by
# construction, so the LSTM-vs-MLP gap is the learning signal.
[[example]]
name = "recurrent_ppo_flickering_cartpole"
path = "examples/games/cartpole/recurrent_ppo_flickering_cartpole.rs"
required-features = ["training"]

# Burst-flickering CartPole — correlated-occlusion POMDP (issue #302). Same
# blank rate p as the i.i.d. flickering env above, but blanks arrive in Markov
# bursts (mean run length ~4) the reactive baseline cannot bridge. Runs LSTM
# and MLP under BOTH regimes and reports the memory gap widening vs. i.i.d.
[[example]]
name = "recurrent_ppo_burst_flickering_cartpole"
path = "examples/games/cartpole/recurrent_ppo_burst_flickering_cartpole.rs"
required-features = ["training"]

# T-maze — the provably memory-hard POMDP (Bakker 2001, issue #302). A cue is
# shown only at step 0; the agent must recall it N steps later at the junction,
# where a memoryless policy is provably at chance. Contrasts LSTM vs MLP across
# a corridor-length sweep N in {5, 10, 20} to locate the memory-horizon knee.
[[example]]
name = "recurrent_ppo_t_maze"
path = "examples/games/t_maze/recurrent_ppo_t_maze.rs"
required-features = ["training"]

# Velocity-masked CartPole — retained as a DOCUMENTED NEGATIVE RESULT (issue
# #287): a 500k-step run showed a memoryless [x, theta] controller solves it
# and beats the LSTM, so velocity-masking alone is not a POMDP. Superseded by
# the flickering example above; kept as the empirical record.
[[example]]
name = "recurrent_ppo_masked_cartpole"
path = "examples/games/cartpole/recurrent_ppo_masked_cartpole.rs"
required-features = ["training"]

# Asynchronous actor-learner CartPole PPO (Phase 2 of the distributed
# training epic #265): N inference-only actor threads stream experiences
# over crossbeam-channel to one PPO learner, which broadcasts policy
# bytes back. Async counterpart of train_cartpole_modern; runnable
# counterpart of tests/test_cartpole_async.rs.
[[example]]
name = "train_cartpole_async"
path = "examples/games/cartpole/train_cartpole_async.rs"
required-features = ["training"]

# DQN on GridWorld (issue #185, PR B of the more-environments epic #180).
# Seeded Double-DQN on the 4x4 FrozenLake-style GridWorld env (16-dim one-hot
# obs, 4 discrete actions). Emits an opt-in learning-curve CSV via CURVE_CSV,
# mirroring train_cartpole_dqn. Runnable counterpart of
# tests/test_dqn_grid_world.rs.
[[example]]
name = "train_dqn_grid_world"
path = "examples/games/grid_world/train_dqn_grid_world.rs"
required-features = ["training"]

# A2C CartPole trainer (issue #153, PR C of the A2C epic #150). Same
# EnvPool rollout + GAE pattern as train_cartpole_modern but with the
# single-update-per-rollout A2cTrainer. Emits an opt-in learning-curve
# CSV via CURVE_CSV for overlay comparison with the PPO example.
[[example]]
name = "train_cartpole_a2c"
path = "examples/games/cartpole/train_cartpole_a2c.rs"
required-features = ["training"]

# End-to-end Behavioral Cloning on CartPole (issue #169, epic #161): train an
# A2C expert, harvest greedy demos, clone a fresh policy with BcTrainer, and
# report cloned-vs-expert mean episode reward. Runnable counterpart of
# tests/test_bc_cartpole.rs. Closes ROADMAP Milestone 6 imitation learning.
[[example]]
name = "train_bc_cartpole"
path = "examples/games/cartpole/train_bc_cartpole.rs"
required-features = ["training"]

# Seeded SAC trainer on the continuous PendulumSwingUp env (issue #162,
# epic #159). Runnable counterpart of tests/test_sac_pendulum.rs; emits
# an opt-in negative-return learning curve via CURVE_CSV.
[[example]]
name = "train_sac"
path = "examples/games/pendulum/train_sac.rs"
required-features = ["training"]

# Seeded SAC trainer on the continuous MountainCarContinuous env (issue
# #168, epic #163). Deceptive-reward sibling of the Pendulum SAC example;
# runnable counterpart of tests/test_sac_mountain_car.rs. Emits an opt-in
# learning curve via CURVE_CSV. The actor action needs no rescaling: the
# env's force range is already [-1, 1].
[[example]]
name = "train_sac_mountain_car"
path = "examples/games/mountain_car/train_sac_mountain_car.rs"
required-features = ["training"]

# Multi-agent / self-play trainers resurrected from the pre-Burn
# deletion (PR #98) per issue #101.
[[example]]
name = "train_snake_multi_v2"
path = "examples/games/snake/train_snake_multi_v2.rs"
required-features = ["training"]

[[example]]
name = "train_pong_self_play"
path = "examples/games/pong/train_pong_self_play.rs"
required-features = ["training"]

# Pong policy evaluation + Burn→InferenceModel exporter. Used to
# generate `pong_self_play_model.json` from the `train_pong_self_play`
# binary record and to compare both Pong models head-to-head against
# the env's built-in rule-based opponent (issue #75 acceptance).
[[example]]
name = "eval_pong"
path = "examples/games/pong/eval_pong.rs"
required-features = ["training"]

# Nature-DQN on ALE Pong via the AtariPreprocess subprocess adapter
# (Epic #306, Phase 4 -- issue #329). `env-atari` is the compile guard;
# add `cuda` (NVIDIA) or `wgpu` (any GPU) at run time to select the GPU
# backend via the `#[cfg]` type alias (see docs/BURN_BACKENDS.md). The
# NdArray smoke run works CPU-only. Requires `ale-py` at run time.
[[example]]
name = "train_pong_dqn"
path = "examples/games/atari/train_pong_dqn.rs"
required-features = ["training", "env-atari"]

# Mixed-precision (f16) Pong DQN (issue #270, epic #267). Same trainer /
# hyperparameters as `train_pong_dqn`, but selects `Autodiff<Cuda<f16,i32>>`
# (or `Wgpu<f16,i32>`) and wraps the backward pass in manual dynamic loss
# scaling when `training-fp16` is on with a GPU backend. `required-features`
# deliberately omits `cuda`/`training-fp16` so `cargo check --all-features`
# type-checks the binary on GPU-less CI; the invalid feature combinations are
# caught by a `compile_error!` inside the source. Verified runtime path: CUDA
# (alc-2 RTX 4090). See docs/PONG_DQN_RUNBOOK.md.
[[example]]
name = "train_pong_dqn_fp16"
path = "examples/games/atari/train_pong_dqn_fp16.rs"
required-features = ["training", "env-atari"]

# Bucket Brigade trainers (issue #115 / PR 4 of #117). Both gated
# behind `env-bucket-brigade` (which is itself feature-gated off in
# the published crate -- see the `[features]` block). The examples
# train PSRO with α-rank and NFSP on the workshop-paper no-convergence
# cells; per-cell selection via `CELL=beta01|beta05|beta09`.
[[example]]
name = "train_psro"
path = "examples/games/bucket_brigade/train_psro.rs"
required-features = ["training", "env-bucket-brigade"]

[[example]]
name = "train_nfsp"
path = "examples/games/bucket_brigade/train_nfsp.rs"
required-features = ["training", "env-bucket-brigade"]

# Standalone single-best-response A/B probe (issue #241): freeze N-1
# opponents, train ONE PPO best-response on one cell, log per-iteration
# val-loss / explained-variance / entropy / mean-return.
[[example]]
name = "train_br_probe"
path = "examples/games/bucket_brigade/train_br_probe.rs"
required-features = ["training", "env-bucket-brigade"]

# SignalingGame protocol-emergence experiment (issue #304): trains the joint
# multi-agent PPO trainer (#295) on the SignalingGame reference env (#292) with
# the comms-loss hook, and measures whether a discrete protocol emerges
# (reward vs a no-comms ablation + referent/message/guess mutual information).
[[example]]
name = "signaling_protocol"
path = "examples/games/signaling/signaling_protocol.rs"
required-features = ["training"]

# Best-response improvability oracle (issue #259): score scripted policies
# (uniform, specialist, firefighter family + random search) for the single BR
# agent vs N-1 frozen uniform opponents; report best achievable per-step team
# return vs the uniform baseline. The non-PPO "improvability gate" (AC#1).
[[example]]
name = "br_oracle"
path = "examples/games/bucket_brigade/br_oracle.rs"
required-features = ["training", "env-bucket-brigade"]

[lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("env-bucket-brigade"))'] }