rusttorch 0.4.0

Build, train, and run neural networks in Rust with LibTorch
Documentation
#!/usr/bin/env python3
from __future__ import annotations

import argparse
from pathlib import Path

import torch
from safetensors.torch import load_file, save_file

from common import (
    ATOL,
    RTOL,
    ResidualModel,
    linear_model,
    reference_input,
    state_data,
    tensor_data,
    write_json,
)


def stepped(optimizer: str) -> dict[str, object]:
    model = linear_model()
    if optimizer == "sgd":
        implementation = torch.optim.SGD(model.parameters(), lr=0.05)
    elif optimizer == "adam":
        implementation = torch.optim.Adam(model.parameters(), lr=0.01)
    elif optimizer == "adamw":
        implementation = torch.optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.1, amsgrad=True)
    elif optimizer == "rmsprop":
        implementation = torch.optim.RMSprop(model.parameters(), lr=0.01, alpha=0.9, weight_decay=0.1, momentum=0.5, centered=True)
    else:
        raise ValueError(f"unknown optimizer: {optimizer}")
    for _ in range(3 if optimizer in ("adamw", "rmsprop") else 1):
        loss = torch.nn.functional.cross_entropy(model(reference_input()), torch.tensor([2, 0]))
        implementation.zero_grad()
        loss.backward()
        implementation.step()
    return {"loss": loss.item(), "state": state_data(model)}



def layer_result(model: torch.nn.Module, value: torch.Tensor) -> dict[str, object]:
    output = model(value)
    output.square().mean().backward()
    result = {
        "forward": tensor_data(output),
        "state": state_data(model),
        "parameter_grads": {
            name: tensor_data(parameter.grad.to_dense() if parameter.grad.is_sparse else parameter.grad)
            for name, parameter in model.named_parameters()
        },
    }
    if value.requires_grad:
        result["input_grad"] = tensor_data(value.grad)
    return result


def layer_examples() -> dict[str, object]:
    result = {}
    for dimensions, shape, layer_type in (
        (1, (2, 2, 7), torch.nn.Conv1d),
        (2, (2, 2, 5, 6), torch.nn.Conv2d),
        (3, (1, 2, 4, 4, 4), torch.nn.Conv3d),
    ):
        torch.manual_seed(900 + dimensions)
        model = layer_type(2, 4, (2,) * dimensions, stride=2, padding=1, dilation=2, groups=2)
        initial = state_data(model)
        with torch.no_grad():
            model.weight.copy_(torch.arange(model.weight.numel()).reshape_as(model.weight) / 10 - 0.2)
            model.bias.copy_(torch.arange(4) / 10)
        value = (torch.arange(torch.tensor(shape).prod().item()).float().reshape(shape) / 20 - 0.5).requires_grad_()
        result[f"conv{dimensions}d"] = {"initial": initial, **layer_result(model, value)}

    model = torch.nn.LayerNorm((2, 3), eps=1e-4, bias=False)
    initial = state_data(model)
    with torch.no_grad():
        model.weight.copy_((torch.arange(6).float() / 10 + 0.5).reshape(2, 3))
    value = (torch.arange(12).float().reshape(2, 2, 3) / 5 - 0.8).requires_grad_()
    result["layer_norm"] = {"initial": initial, **layer_result(model, value)}

    for sparse in (False, True):
        torch.manual_seed(902)
        model = torch.nn.Embedding(5, 3, padding_idx=-1, scale_grad_by_freq=not sparse, sparse=sparse)
        initial = state_data(model)
        with torch.no_grad():
            model.weight.copy_((torch.arange(15).float() / 10).reshape(5, 3))
        value = torch.tensor([[0, 1, 1], [4, 2, 4]])
        result["embedding_sparse" if sparse else "embedding"] = {"initial": initial, **layer_result(model, value)}
    return result


def generate(output_dir: Path) -> None:
    output_dir.mkdir(parents=True, exist_ok=True)
    torch.manual_seed(0)

    model = linear_model()
    value = reference_input(requires_grad=True)
    output = model(value)
    linear_loss = output.square().mean()
    linear_loss.backward()

    logits = model(reference_input()).detach()
    cross_entropy = torch.nn.functional.cross_entropy(logits, torch.tensor([2, 0]))
    mse_target = torch.tensor([[0.0, 0.5, -0.5], [1.0, -1.0, 0.25]])
    mse = torch.nn.functional.mse_loss(logits, mse_target)

    residual = ResidualModel()
    residual_input = reference_input(requires_grad=True)
    residual_output = residual(residual_input)
    residual_loss = residual_output.square().mean()
    residual_loss.backward()

    safetensors_path = output_dir / "pytorch_linear.safetensors"
    save_file(model.state_dict(), safetensors_path)
    loaded = load_file(safetensors_path)
    for name, expected in model.state_dict().items():
        torch.testing.assert_close(loaded[name], expected, rtol=0, atol=0)

    reference = {
        "metadata": {
            "schema": 1,
            "torch": torch.__version__,
            "dtype": "float32",
            "rtol": RTOL,
            "atol": ATOL,
        },
        "input": tensor_data(reference_input()),
        "linear": {
            "forward": tensor_data(output),
            "loss": linear_loss.item(),
            "input_grad": tensor_data(value.grad),
            "weight_grad": tensor_data(model.weight.grad),
            "bias_grad": tensor_data(model.bias.grad),
        },
        "losses": {"cross_entropy": cross_entropy.item(), "mse": mse.item()},
        "sgd": stepped("sgd"),
        "adam": stepped("adam"),
        "adamw": stepped("adamw"),
        "rmsprop": stepped("rmsprop"),
        "layers": layer_examples(),
        "residual": {
            "forward": tensor_data(residual_output),
            "loss": residual_loss.item(),
            "input_grad": tensor_data(residual_input.grad),
            "parameter_grads": {
                name: tensor_data(parameter.grad) for name, parameter in residual.named_parameters()
            },
        },
        "safetensors": {"file": safetensors_path.name, "keys": sorted(loaded)},
    }
    write_json(output_dir / "reference.json", reference)
    write_json(
        output_dir / "linear_io.json",
        {"input": reference["input"], "output": reference["linear"]["forward"]},
    )
    print(output_dir / "reference.json")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Generate deterministic PyTorch parity assets")
    parser.add_argument("output_dir", type=Path)
    generate(parser.parse_args().output_dir)