rusttorch 0.4.0

Build, train, and run neural networks in Rust with LibTorch
Documentation
use std::{
    collections::BTreeMap,
    fs,
    path::{Path, PathBuf},
    process::Command,
};

use rusttorch::{
    Device, Kind, Result, Tensor,
    interop::{load_state_dict, save_state_dict},
    nn, optim,
};
use serde_json::{Value, json};
use tch::nn::VarStore;

#[test]
#[ignore = "run through scripts/run-python-parity.sh"]
fn bidirectional_python_parity() -> Result<()> {
    let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    let reference_dir = PathBuf::from(
        std::env::var_os("RUSTTORCH_PYTHON_REFERENCE_DIR")
            .expect("run this ignored test through scripts/run-python-parity.sh"),
    );
    let reference: Value = serde_json::from_slice(
        &fs::read(reference_dir.join("reference.json")).expect("read Python reference JSON"),
    )
    .expect("parse Python reference JSON");
    let input = tensor_from_json(&reference["input"], true);
    let weights = reference_dir.join("pytorch_linear.safetensors");

    let (var_store, layer) = loaded_linear(&weights)?;
    let output = layer.forward(&input)?;
    let linear_loss = output.f_square()?.f_mean(Kind::Float)?;
    linear_loss.backward();

    let targets = Tensor::from_slice(&[2i64, 0]);
    let plain_input = tensor_from_json(&reference["input"], false);
    let logits = layer.forward(&plain_input)?;
    let cross_entropy = nn::functional::cross_entropy(&logits, &targets)?;
    let mse_target = Tensor::from_slice(&[0.0f32, 0.5, -0.5, 1.0, -1.0, 0.25]).view([2, 3]);
    let mse = nn::functional::mse_loss(&logits, &mse_target)?;

    let linear = json!({
        "forward": tensor_json(&output),
        "loss": linear_loss.double_value(&[]),
        "input_grad": tensor_json(&input.grad()),
        "weight_grad": tensor_json(&layer.weight().grad()),
        "bias_grad": tensor_json(&layer.bias().expect("linear bias").grad()),
    });
    let sgd = optimizer_step(&weights, &reference, OptimizerKind::Sgd)?;
    let adam = optimizer_step(&weights, &reference, OptimizerKind::Adam)?;
    let residual = residual_result(&reference)?;
    let adamw = optimizer_step(&weights, &reference, OptimizerKind::AdamW)?;
    let rmsprop = optimizer_step(&weights, &reference, OptimizerKind::RmsProp)?;
    let layers = layer_examples()?;

    let rust_results = json!({
        "linear": linear,
        "losses": {
            "cross_entropy": cross_entropy.double_value(&[]),
            "mse": mse.double_value(&[]),
        },
        "sgd": sgd,
        "adam": adam,
        "residual": residual,
        "adamw": adamw,
        "rmsprop": rmsprop,
        "layers": layers,
    });
    let rust_results_path = reference_dir.join("rust_results.json");
    fs::write(
        &rust_results_path,
        serde_json::to_vec_pretty(&rust_results).expect("serialize Rust parity results"),
    )
    .expect("write Rust parity results");

    let rust_weights = reference_dir.join("rusttorch_linear.safetensors");
    save_state_dict(&rust_weights, &var_store)?;
    let rust_io = reference_dir.join("rust_linear_io.json");
    fs::write(
        &rust_io,
        serde_json::to_vec_pretty(&json!({
            "input": reference["input"],
            "output": tensor_json(&output),
        }))
        .expect("serialize Rust linear IO"),
    )
    .expect("write Rust linear IO");

    run_python(
        &root,
        "scripts/verify-rusttorch-load.py",
        &[&rust_results_path, &reference_dir.join("reference.json")],
    );
    run_python(
        &root,
        "scripts/verify-pytorch-load.py",
        &[&rust_weights, &rust_io],
    );
    Ok(())
}

fn loaded_linear(weights: &Path) -> Result<(VarStore, nn::Linear)> {
    let var_store = VarStore::new(Device::Cpu);
    let layer = nn::LinearConfig::new(4, 3).build(&var_store.root())?;
    let report = load_state_dict(weights, &var_store)?;
    assert_eq!(report.loaded, ["bias", "weight"]);
    Ok((var_store, layer))
}

#[derive(Clone, Copy)]
enum OptimizerKind {
    Sgd,
    Adam,
    AdamW,
    RmsProp,
}

fn optimizer_step(weights: &Path, reference: &Value, kind: OptimizerKind) -> Result<Value> {
    let (var_store, layer) = loaded_linear(weights)?;
    let input = tensor_from_json(&reference["input"], false);
    let target = Tensor::from_slice(&[2i64, 0]);
    let mut optimizer = match kind {
        OptimizerKind::Sgd => optim::Sgd::builder()
            .learning_rate(0.05)
            .build(&var_store)?,
        OptimizerKind::Adam => optim::Adam::builder()
            .learning_rate(0.01)
            .build(&var_store)?,
        OptimizerKind::AdamW => optim::AdamW::builder()
            .learning_rate(0.01)
            .weight_decay(0.1)
            .amsgrad(true)
            .build(&var_store)?,
        OptimizerKind::RmsProp => optim::RmsProp::builder()
            .learning_rate(0.01)
            .alpha(0.9)
            .weight_decay(0.1)
            .momentum(0.5)
            .centered(true)
            .build(&var_store)?,
    };
    let mut last_loss = 0.0;
    for _ in 0..if matches!(kind, OptimizerKind::AdamW | OptimizerKind::RmsProp) {
        3
    } else {
        1
    } {
        let loss = nn::functional::cross_entropy(&layer.forward(&input)?, &target)?;
        last_loss = loss.double_value(&[]);
        optimizer.backward_step(&loss)?;
    }
    Ok(json!({
        "loss": last_loss,
        "state": state_json(&var_store),
    }))
}

fn layer_result(module: &impl nn::Module, store: &VarStore, input: &Tensor) -> Result<Value> {
    let output = module.forward(input)?;
    output.f_square()?.f_mean(Kind::Float)?.f_backward()?;
    let gradients = store
        .variables()
        .into_iter()
        .map(|(name, tensor)| {
            let gradient = tensor.grad();
            let gradient = if gradient.is_sparse() {
                gradient.to_dense(None, false)
            } else {
                gradient
            };
            (name, tensor_json(&gradient))
        })
        .collect::<BTreeMap<_, _>>();
    let mut result = json!({
        "forward": tensor_json(&output),
        "state": state_json(store),
        "parameter_grads": gradients,
    });
    if input.requires_grad() {
        result["input_grad"] = tensor_json(&input.grad());
    }
    Ok(result)
}

fn convolution_example<const D: usize>(shape: &[i64]) -> Result<Value> {
    rusttorch::manual_seed(900 + D as i64);
    let store = VarStore::new(Device::Cpu);
    let layer = nn::ConvConfig::new(2, 4, [2; D])
        .stride([2; D])
        .padding([1; D])
        .dilation([2; D])
        .groups(2)
        .build(&store.root())?;
    let initial = state_json(&store);
    let weight = (Tensor::arange(layer.weight().numel() as i64, (Kind::Float, Device::Cpu)) / 10.
        - 0.2)
        .reshape(layer.weight().size());
    let bias = Tensor::arange(4, (Kind::Float, Device::Cpu)) / 10.;
    rusttorch::no_grad(|| -> Result<()> {
        layer.weight().shallow_clone().f_copy_(&weight)?;
        layer.bias().unwrap().shallow_clone().f_copy_(&bias)?;
        Ok(())
    })?;
    let input = (Tensor::arange(shape.iter().product::<i64>(), (Kind::Float, Device::Cpu))
        .reshape(shape)
        / 20.
        - 0.5)
        .set_requires_grad(true);
    let mut result = layer_result(&layer, &store, &input)?;
    result["initial"] = initial;
    Ok(result)
}

fn layer_examples() -> Result<Value> {
    let mut result = json!({
        "conv1d": convolution_example::<1>(&[2, 2, 7])?,
        "conv2d": convolution_example::<2>(&[2, 2, 5, 6])?,
        "conv3d": convolution_example::<3>(&[1, 2, 4, 4, 4])?,
    });
    let store = VarStore::new(Device::Cpu);
    let layer = nn::LayerNormConfig::new([2, 3])
        .eps(1e-4)
        .bias(false)
        .build(&store.root())?;
    let initial = state_json(&store);
    let weight = (Tensor::arange(6, (Kind::Float, Device::Cpu)) / 10. + 0.5).reshape([2, 3]);
    rusttorch::no_grad(|| layer.weight().unwrap().shallow_clone().f_copy_(&weight))?;
    let input = (Tensor::arange(12, (Kind::Float, Device::Cpu)).reshape([2, 2, 3]) / 5. - 0.8)
        .set_requires_grad(true);
    result["layer_norm"] = layer_result(&layer, &store, &input)?;
    result["layer_norm"]["initial"] = initial;

    for sparse in [false, true] {
        rusttorch::manual_seed(902);
        let store = VarStore::new(Device::Cpu);
        let layer = nn::EmbeddingConfig::new(5, 3)
            .padding_idx(-1)
            .scale_grad_by_freq(!sparse)
            .sparse(sparse)
            .build(&store.root())?;
        let initial = state_json(&store);
        let weight = (Tensor::arange(15, (Kind::Float, Device::Cpu)) / 10.).reshape([5, 3]);
        rusttorch::no_grad(|| layer.weight().shallow_clone().f_copy_(&weight))?;
        let input = Tensor::from_slice(&[0_i64, 1, 1, 4, 2, 4]).reshape([2, 3]);
        let key = if sparse {
            "embedding_sparse"
        } else {
            "embedding"
        };
        result[key] = layer_result(&layer, &store, &input)?;
        result[key]["initial"] = initial;
    }
    Ok(result)
}

fn residual_result(reference: &Value) -> Result<Value> {
    let var_store = VarStore::new(Device::Cpu);
    let main = nn::LinearConfig::new(4, 4)
        .bias(false)
        .build(&(var_store.root() / "main"))?;
    let skip = nn::LinearConfig::new(4, 4)
        .bias(false)
        .build(&(var_store.root() / "skip"))?;
    assign(
        main.weight(),
        &[
            0.5, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.0, 0.0, 0.0, 0.5, 0.0, 0.0, 0.0, 0.0, 0.5,
        ],
        [4, 4],
    )?;
    assign(
        skip.weight(),
        &[
            0.0, 0.1, 0.0, -0.2, 0.3, 0.0, 0.2, 0.0, 0.0, -0.4, 0.0, 0.1, 0.2, 0.0, -0.3, 0.0,
        ],
        [4, 4],
    )?;
    let input = tensor_from_json(&reference["input"], true);
    let output = nn::functional::relu(&main.forward(&input)?)?.f_add(&skip.forward(&input)?)?;
    let loss = output.f_square()?.f_mean(Kind::Float)?;
    loss.backward();
    Ok(json!({
        "forward": tensor_json(&output),
        "loss": loss.double_value(&[]),
        "input_grad": tensor_json(&input.grad()),
        "parameter_grads": {
            "main.weight": tensor_json(&main.weight().grad()),
            "skip.weight": tensor_json(&skip.weight().grad()),
        },
    }))
}

fn assign<const N: usize>(tensor: &Tensor, values: &[f32], shape: [i64; N]) -> Result<()> {
    let source = Tensor::from_slice(values).view(shape.as_slice());
    let mut destination = tensor.shallow_clone();
    rusttorch::no_grad(|| destination.f_copy_(&source))?;
    Ok(())
}

fn state_json(var_store: &VarStore) -> Value {
    var_store
        .variables()
        .into_iter()
        .collect::<BTreeMap<_, _>>()
        .into_iter()
        .map(|(name, tensor)| (name, tensor_json(&tensor)))
        .collect()
}

fn tensor_from_json(value: &Value, requires_grad: bool) -> Tensor {
    let rows = value.as_array().expect("tensor JSON rows");
    let columns = rows
        .first()
        .expect("non-empty tensor")
        .as_array()
        .expect("tensor JSON columns");
    let flat = rows
        .iter()
        .flat_map(|row| row.as_array().expect("tensor JSON row"))
        .map(|value| value.as_f64().expect("numeric tensor JSON") as f32)
        .collect::<Vec<_>>();
    Tensor::from_slice(&flat)
        .view([rows.len() as i64, columns.len() as i64])
        .set_requires_grad(requires_grad)
}

fn tensor_json(tensor: &Tensor) -> Value {
    match tensor.size().len() {
        0 => json!(tensor.double_value(&[])),
        1 => json!(Vec::<f32>::try_from(tensor).expect("convert rank-1 tensor")),
        2 => json!(Vec::<Vec<f32>>::try_from(tensor).expect("convert rank-2 tensor")),
        _ => json!(tensor.unbind(0).iter().map(tensor_json).collect::<Vec<_>>()),
    }
}

fn run_python(root: &Path, script: &str, arguments: &[&Path]) {
    let python = root.join(".venv/bin/python");
    let status = Command::new(python)
        .arg(root.join(script))
        .args(arguments)
        .status()
        .expect("run Python parity verifier");
    assert!(status.success(), "Python verifier {script} failed");
}