unity_formats/

program.rs

//! A Shader's compiled programs, as a player stores them: per pass, each stage's program (vertex, fragment, geometry,
//! hull, domain: the stages the pass has) with its variants (each a keyword
//! set, a GPU program type and an entry of the compressed blob) and their parameters (constant buffers with each
//! field's name and byte offset, texture and sampler bind slots), read from `m_ParsedForm`; and a variant's code from
//! the platform's LZ4-compressed blob. What the bindings mean for a material is the caller's.
//!
//! Players store the variants in one of two layouts, told apart by their fields: from 2021.2, `m_PlayerSubPrograms`
//! (lists of variants, keywords indexing the form's `m_KeywordNames`) with `m_CommonParameters` shared by them;
//! before it, `m_SubPrograms` (one list, each variant with its hardware tier, its keywords as global and local indices
//! into the pass's name table, and its own parameters). Struct fields (instancing's per-draw arrays) are read as
//! [`StructField`]s.
//!
//! The blob of one platform is split into segments (`offsets`, `compressedLengths`, `decompressedLengths`, one
//! entry per segment). Segment 0 starts with the entry table: a count, then per entry its offset, length and segment.
//! An entry of a program holds Unity's program record: its version, GPU program type, then (from 2016.2) a header the
//! reader skips, keyword strings (empty in current players: the parsed form names them), and the code as a byte array.

use primase_foundations::compress::{Decompressor, Lz4};

use crate::player::{ObjRef, Player};
use crate::value::{Array, Value};
use crate::{Result, invalid};

/// A constant-buffer field: its name, byte offset in the buffer, element type (Unity's `ShaderParamType`: 0 float, 1
/// int, 2 bool, 3 half, 4 short, 5 uint), shape (a vector of `columns` lanes, or a matrix of `rows` rows) and array
/// size (0: not an array).
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
pub struct Field {
    /// The field's name.
    pub name: String,
    /// The field's byte offset in its buffer.
    pub offset: u32,
    /// Unity's `ShaderParamType` of the element.
    pub kind: i64,
    /// A vector or a matrix.
    pub shape: Shape,
    /// The array length; 0 when the field is not an array.
    pub array_size: u32,
}

/// A field's shape.
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)]
pub enum Shape {
    /// A vector.
    Vector {
        /// Its lane count (the stored column count: 2 to 4).
        columns: u32,
    },
    /// A matrix.
    Matrix {
        /// Its row count as stored.
        rows: u32,
    },
}

/// A constant-buffer field of a struct type (Unity's instancing arrays): its name, byte offset, array size (0: not
/// an array), the struct's size in bytes and its members (offsets within the struct).
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
pub struct StructField {
    /// The field's name.
    pub name: String,
    /// The field's byte offset in its buffer.
    pub offset: u32,
    /// The array length; 0 when the field is not an array.
    pub array_size: u32,
    /// The struct's size in bytes.
    pub size: u32,
    /// The struct's members, their offsets within the struct.
    pub members: Vec<Field>,
}

/// A constant buffer: its name, the bind slot it is bound to (`b<n>`; `None` when no binding names it), its size in
/// bytes, its fields and its struct fields (by offset).
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
pub struct ConstantBuffer {
    /// The buffer's name.
    pub name: String,
    /// The bind slot (`b<n>`), when a binding names the buffer.
    pub slot: Option<u32>,
    /// The buffer's size in bytes.
    pub size: u32,
    /// The buffer's fields.
    pub fields: Vec<Field>,
    /// The buffer's struct fields, by offset.
    #[serde(skip_serializing_if = "Vec::is_empty")]
    pub structs: Vec<StructField>,
}

/// A texture parameter: its name, bind slot (`t<n>`), the sampler slot it samples with (`None`: none of its own) and
/// dimension (Unity's `TextureDimension`: 2 a 2D texture, 3 3D, 4 a cube).
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
pub struct TextureParam {
    /// The texture's name.
    pub name: String,
    /// The bind slot (`t<n>`).
    pub slot: u32,
    /// The slot of the sampler it samples with; `None` when it has none of its own.
    pub sampler: Option<u32>,
    /// Unity's `TextureDimension`: 2 a 2D texture, 3 a 3D texture, 4 a cube.
    pub dim: i64,
}

/// The parameters the variants of a program share.
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
pub struct Parameters {
    /// The constant buffers.
    pub constant_buffers: Vec<ConstantBuffer>,
    /// The texture parameters.
    pub textures: Vec<TextureParam>,
    /// Sampler states: Unity's packed sampler value and its bind slot (`s<n>`).
    pub samplers: Vec<(u32, u32)>,
}

/// One compiled variant: the blob entry holding its code, its keywords, its GPU program type (Unity's
/// `ShaderGpuProgramType`: 17 a D3D11 pixel program before 2021.2, 18 an SM 5.0 one after) and its shader
/// requirements mask; `group` is the index of the player sub-program list it is in (from 2021.2) or its hardware tier
/// (before); `parameters` its own, where the player stores them per variant (before 2021.2), else `None`: the pass's.
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
pub struct Variant {
    /// The index of the player sub-program list the variant is in (2021.2 on), or its hardware tier (before).
    pub group: usize,
    /// The entry of the platform's blob that holds the variant's program record.
    pub blob_index: u32,
    /// The keywords the variant is compiled with.
    pub keywords: Vec<String>,
    /// Unity's `ShaderGpuProgramType`.
    pub gpu_program_type: i64,
    /// The shader requirements mask.
    pub requirements: i64,
    /// The blob entry holding this variant's own parameters, where the player stores them there (from 2021.2:
    /// `m_ParameterBlobIndices`, parallel to the sub-program lists).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub parameter_blob: Option<u32>,
    /// The variant's own parameters, where the player stores them per variant (before 2021.2); `None`: the pass
    /// program's.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub parameters: Option<Parameters>,
}

/// One pass's program of one stage: its variants and the parameters common to them (empty where each variant has its
/// own).
#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)]
pub struct PassProgram {
    /// The variants, in stored order.
    pub variants: Vec<Variant>,
    /// The parameters the variants share.
    pub parameters: Parameters,
}

impl PassProgram {
    /// The parameters variant `v` (of this pass) is compiled with.
    pub fn parameters_of<'a>(&'a self, v: &'a Variant) -> &'a Parameters {
        v.parameters.as_ref().unwrap_or(&self.parameters)
    }
}

/// The stages Unity stores a pass's programs for (`progVertex`, `progFragment`, `progGeometry`, `progHull`,
/// `progDomain`), in that order.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
pub enum Stage {
    /// `progVertex`.
    Vertex,
    /// `progFragment`.
    Fragment,
    /// `progGeometry`.
    Geometry,
    /// `progHull`.
    Hull,
    /// `progDomain`.
    Domain,
}

impl Stage {
    /// Every stage, in Unity's order.
    pub const ALL: [Stage; 5] = [
        Stage::Vertex,
        Stage::Fragment,
        Stage::Geometry,
        Stage::Hull,
        Stage::Domain,
    ];

    /// The pass field Unity stores the stage's program under.
    fn field(self) -> &'static str {
        match self {
            Stage::Vertex => "progVertex",
            Stage::Fragment => "progFragment",
            Stage::Geometry => "progGeometry",
            Stage::Hull => "progHull",
            Stage::Domain => "progDomain",
        }
    }
}

/// One pass's programs, a [`PassProgram`] per stage (empty where the pass stores none for it).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct PassStages {
    /// The vertex program.
    pub vertex: PassProgram,
    /// The fragment program.
    pub fragment: PassProgram,
    /// The geometry program.
    pub geometry: PassProgram,
    /// The hull program.
    pub hull: PassProgram,
    /// The domain program.
    pub domain: PassProgram,
}

impl PassStages {
    /// The program of `stage`.
    pub fn stage(&self, stage: Stage) -> &PassProgram {
        match stage {
            Stage::Vertex => &self.vertex,
            Stage::Fragment => &self.fragment,
            Stage::Geometry => &self.geometry,
            Stage::Hull => &self.hull,
            Stage::Domain => &self.domain,
        }
    }
}

/// A Shader's compiled programs: `passes[subshader][pass]`, and its per-platform blobs.
#[derive(Clone, Debug, PartialEq)]
pub struct ShaderPrograms {
    /// The shader's name (`m_ParsedForm.m_Name`).
    pub name: String,
    /// The passes of each subshader.
    pub passes: Vec<Vec<PassStages>>,
    /// Unity's `ShaderCompilerPlatform` of each blob (4: D3D11).
    pub platforms: Vec<i64>,
    /// Per platform, per segment: offset, compressed and decompressed length.
    segments: Vec<Vec<(usize, usize, usize)>>,
    blob: Vec<u8>,
}

fn items<'a>(v: &'a Value, k: &str) -> Result<&'a [Value]> {
    match v.get(k) {
        Some(Value::Array(Array::Values(items))) => Ok(items),
        Some(Value::Array(a)) if a.is_empty() => Ok(&[]),
        None => Ok(&[]),
        _ => Err(invalid(format!("shader program: {k} is not a list"))),
    }
}

/// An array of integers, whatever element type the type tree gave it.
fn ints(v: &Value) -> Result<Vec<i64>> {
    let bad = || invalid("shader program: an integer array of another type");
    Ok(match v {
        Value::Array(a) => match a {
            Array::I8(x) => x.iter().map(|&n| i64::from(n)).collect(),
            Array::U8(x) => x.iter().map(|&n| i64::from(n)).collect(),
            Array::I16(x) => x.iter().map(|&n| i64::from(n)).collect(),
            Array::U16(x) => x.iter().map(|&n| i64::from(n)).collect(),
            Array::I32(x) => x.iter().map(|&n| i64::from(n)).collect(),
            Array::U32(x) => x.iter().map(|&n| i64::from(n)).collect(),
            Array::I64(x) => x.clone(),
            Array::U64(x) => x
                .iter()
                .map(|&n| i64::try_from(n).map_err(|_| bad()))
                .collect::<Result<_>>()?,
            Array::Values(x) => x
                .iter()
                .map(|n| n.as_i64().ok_or_else(bad))
                .collect::<Result<_>>()?,
            _ => return Err(bad()),
        },
        _ => return Err(bad()),
    })
}

fn int(v: &Value, k: &str) -> Result<i64> {
    v.get(k)
        .and_then(Value::as_i64)
        .ok_or_else(|| invalid(format!("shader program: no integer {k}")))
}

fn u32_of(n: i64, what: &str) -> Result<u32> {
    u32::try_from(n).map_err(|_| invalid(format!("shader program: {what} {n} out of range")))
}

/// The name an entry's `m_NameIndex` gives in the pass's name table.
fn name_of(e: &Value, names: &std::collections::BTreeMap<i64, String>) -> Result<String> {
    let i = int(e, "m_NameIndex")?;
    names
        .get(&i)
        .cloned()
        .ok_or_else(|| invalid(format!("shader program: name index {i} past the pass's names")))
}

/// The vector (`vectors`) and matrix (`matrices`) fields of a constant buffer or struct, by offset.
fn fields(
    v: &Value,
    vectors: &str,
    matrices: &str,
    names: &std::collections::BTreeMap<i64, String>,
) -> Result<Vec<Field>> {
    let mut fields = Vec::new();
    for (key, matrix) in [(vectors, false), (matrices, true)] {
        for f in items(v, key)? {
            fields.push(Field {
                name: name_of(f, names)?,
                offset: u32_of(int(f, "m_Index")?, "field offset")?,
                kind: int(f, "m_Type")?,
                shape: if matrix {
                    Shape::Matrix {
                        rows: u32_of(int(f, "m_RowCount")?, "matrix rows")?,
                    }
                } else {
                    Shape::Vector {
                        columns: u32_of(int(f, "m_Dim")?, "vector size")?,
                    }
                },
                array_size: u32_of(int(f, "m_ArraySize")?, "array size")?,
            });
        }
    }
    fields.sort_by_key(|f| f.offset);
    Ok(fields)
}

/// The parameters as stored, their names resolved through the pass's name table.
fn parameters(v: &Value, names: &std::collections::BTreeMap<i64, String>) -> Result<Parameters> {
    let name = |e: &Value| name_of(e, names);
    let bindings = items(v, "m_ConstantBufferBindings")?
        .iter()
        .map(|b| {
            Ok((
                int(b, "m_NameIndex")?,
                u32_of(int(b, "m_Index")?, "binding slot")?,
            ))
        })
        .collect::<Result<std::collections::BTreeMap<i64, u32>>>()?;
    let mut constant_buffers = Vec::new();
    for cb in items(v, "m_ConstantBuffers")? {
        let mut structs = items(cb, "m_StructParams")?
            .iter()
            .map(|s| {
                Ok(StructField {
                    name: name(s)?,
                    offset: u32_of(int(s, "m_Index")?, "struct offset")?,
                    array_size: u32_of(int(s, "m_ArraySize")?, "array size")?,
                    size: u32_of(int(s, "m_StructSize")?, "struct size")?,
                    members: fields(s, "m_VectorMembers", "m_MatrixMembers", names)?,
                })
            })
            .collect::<Result<Vec<_>>>()?;
        structs.sort_by_key(|s| s.offset);
        constant_buffers.push(ConstantBuffer {
            name: name(cb)?,
            slot: bindings.get(&int(cb, "m_NameIndex")?).copied(),
            size: u32_of(int(cb, "m_Size")?, "buffer size")?,
            fields: fields(cb, "m_VectorParams", "m_MatrixParams", names)?,
            structs,
        });
    }
    let textures = items(v, "m_TextureParams")?
        .iter()
        .map(|t| {
            let sampler = int(t, "m_SamplerIndex")?;
            Ok(TextureParam {
                name: name(t)?,
                slot: u32_of(int(t, "m_Index")?, "texture slot")?,
                sampler: (sampler >= 0)
                    .then(|| u32_of(sampler, "sampler slot"))
                    .transpose()?,
                dim: int(t, "m_Dim")?,
            })
        })
        .collect::<Result<_>>()?;
    let samplers = items(v, "m_Samplers")?
        .iter()
        .map(|s| {
            Ok((
                u32_of(int(s, "sampler")?, "sampler state")?,
                u32_of(int(s, "bindPoint")?, "sampler slot")?,
            ))
        })
        .collect::<Result<_>>()?;
    Ok(Parameters {
        constant_buffers,
        textures,
        samplers,
    })
}

/// The first word of a parameter blob entry in a 2021.2 player.
const PARAMETER_BLOB_MAGIC: u32 = 0x0C0A_75BA;

/// A length-prefixed string of a parameter blob: its length, its bytes, padding to four.
fn blob_string(r: &mut crate::reader::Reader<'_>) -> Result<String> {
    let n = usize::try_from(r.u32()?).map_err(|_| invalid("parameter blob: a string length"))?;
    let bytes = r.take(n)?;
    r.align(4)?;
    String::from_utf8(bytes.to_vec()).map_err(|_| invalid("parameter blob: a name that is not UTF-8"))
}

/// A field of a constant buffer or struct: its name, then type, rows, columns, whether it is a matrix, array size and
/// byte offset (each a word).
fn blob_field(r: &mut crate::reader::Reader<'_>) -> Result<Field> {
    let name = blob_string(r)?;
    let kind = i64::from(r.u32()?);
    let (rows, columns, matrix) = (r.u32()?, r.u32()?, r.u32()?);
    let array_size = r.u32()?;
    let offset = r.u32()?;
    let shape = match matrix {
        0 => Shape::Vector { columns },
        1 => Shape::Matrix { rows },
        n => {
            return Err(invalid(format!(
                "parameter blob: field {name} has matrix flag {n}"
            )));
        }
    };
    Ok(Field {
        name,
        offset,
        kind,
        shape,
        array_size,
    })
}

/// A variant's own parameters as a 2021.2 player stores them in the blob entry `m_ParameterBlobIndices` names:
/// the magic word, a count of the constant buffers plus one, four zero words, then each constant buffer (name, size,
/// its fields, its structs: name, offset, array size, size and fields) and a list of entries, each a name, a kind and
/// a slot: kind 0 a texture (the sampler slot, `u32::MAX` for none, and a dimension word), kind 1 a constant buffer's
/// binding (a word, 0), kind 4 a sampler (its packed state; the name is empty). The layout was held to every one of
/// the 3,280 entries of Sledding Game's `level1` shaders (3,272 read to the last byte; the 8 refused hold a kind 2,
/// a structured buffer of the DOTS instancing, whose payload is not read).
pub fn parameter_blob(entry: &[u8]) -> Result<Parameters> {
    let mut r = crate::reader::Reader::new(entry, false);
    if r.u32()? != PARAMETER_BLOB_MAGIC {
        return Err(invalid("parameter blob: not the 2021.2 format"));
    }
    let buffers = r.u32()?;
    if (0..4).any(|_| r.u32().map_or(true, |w| w != 0)) {
        return Err(invalid(
            "parameter blob: words before the buffers that are not zero",
        ));
    }
    let mut constant_buffers = Vec::new();
    // the buffers are the entries up to the list's count word: a buffer's name is a string, the count is not one
    for _ in 1..buffers {
        let name = blob_string(&mut r)?;
        let size = r.u32()?;
        let fields = (0..r.u32()?)
            .map(|_| blob_field(&mut r))
            .collect::<Result<Vec<_>>>()?;
        let mut structs = Vec::new();
        for _ in 0..r.u32()? {
            structs.push(StructField {
                name: blob_string(&mut r)?,
                offset: r.u32()?,
                array_size: r.u32()?,
                size: r.u32()?,
                members: (0..r.u32()?)
                    .map(|_| blob_field(&mut r))
                    .collect::<Result<Vec<_>>>()?,
            });
        }
        constant_buffers.push(ConstantBuffer {
            name,
            slot: None,
            size,
            fields,
            structs,
        });
    }
    let (mut textures, mut samplers) = (Vec::new(), Vec::new());
    for _ in 0..r.u32()? {
        let name = blob_string(&mut r)?;
        let (kind, slot) = (r.u32()?, r.u32()?);
        match kind {
            0 => {
                let sampler = r.u32()?;
                textures.push(TextureParam {
                    name,
                    slot,
                    sampler: (sampler != u32::MAX).then_some(sampler),
                    dim: i64::from(r.u32()?),
                });
            }
            1 => {
                r.u32()?;
                // a buffer the blob binds and holds none of is the pass's common one: an empty buffer carries the slot
                match constant_buffers.iter_mut().find(|cb| cb.name == name) {
                    Some(cb) => cb.slot = Some(slot),
                    None => constant_buffers.push(ConstantBuffer {
                        name,
                        slot: Some(slot),
                        size: 0,
                        fields: Vec::new(),
                        structs: Vec::new(),
                    }),
                }
            }
            4 => samplers.push((r.u32()?, slot)),
            n => return Err(invalid(format!("parameter blob: an entry of kind {n}"))),
        }
    }
    if r.remaining() != 0 {
        return Err(invalid("parameter blob: bytes after the entries"));
    }
    Ok(Parameters {
        constant_buffers,
        textures,
        samplers,
    })
}

/// A variant's parameters from the pass's common ones and its own blob: the buffers by name (fields and structs of
/// both, the blob's taking precedence; the larger size; the blob's slot, else the common one), then the textures and
/// samplers of both.
pub fn merge_parameters(common: &Parameters, own: &Parameters) -> Parameters {
    let mut buffers = own.constant_buffers.clone();
    for c in &common.constant_buffers {
        match buffers.iter_mut().find(|b| b.name == c.name) {
            Some(b) => {
                for f in &c.fields {
                    if !b.fields.iter().any(|g| g.name == f.name && g.offset == f.offset) {
                        b.fields.push(f.clone());
                    }
                }
                b.fields.sort_by_key(|f| f.offset);
                for s in &c.structs {
                    if !b.structs.iter().any(|g| g.name == s.name) {
                        b.structs.push(s.clone());
                    }
                }
                b.structs.sort_by_key(|s| s.offset);
                b.size = b.size.max(c.size);
                b.slot = b.slot.or(c.slot);
            }
            None => buffers.push(c.clone()),
        }
    }
    let mut textures = own.textures.clone();
    textures.extend(
        common
            .textures
            .iter()
            .filter(|t| !own.textures.iter().any(|o| o.slot == t.slot))
            .cloned(),
    );
    let mut samplers = own.samplers.clone();
    samplers.extend(
        common
            .samplers
            .iter()
            .filter(|s| !own.samplers.contains(s))
            .copied(),
    );
    Parameters {
        constant_buffers: buffers,
        textures,
        samplers,
    }
}

/// One stage's program of a pass as stored (`program` is its `prog*` value): its variants and common parameters, names
/// resolved through the pass's name table `names` and the form's `keyword_names`; `shader` names the shader in errors.
fn stage_program(
    program: &Value,
    names: &std::collections::BTreeMap<i64, String>,
    keyword_names: &[String],
    name: &str,
) -> Result<PassProgram> {
    let mut variants = Vec::new();
    // before 2021.2: one list, each variant with its tier, keywords by name index and own parameters
    for v in items(program, "m_SubPrograms")? {
        let mut keywords = Vec::new();
        for k in ["m_GlobalKeywordIndices", "m_LocalKeywordIndices"] {
            if let Some(k) = v.get(k) {
                for i in ints(k)? {
                    keywords.push(names.get(&i).cloned().ok_or_else(|| {
                        invalid(format!("shader {name}: keyword index {i} past the pass's names"))
                    })?);
                }
            }
        }
        variants.push(Variant {
            group: usize::try_from(int(v, "m_ShaderHardwareTier")?)
                .map_err(|_| invalid("shader program: a negative hardware tier"))?,
            blob_index: u32_of(int(v, "m_BlobIndex")?, "blob index")?,
            keywords,
            gpu_program_type: int(v, "m_GpuProgramType")?,
            requirements: v.get("m_ShaderRequirements").and_then(Value::as_i64).unwrap_or(0),
            parameter_blob: None,
            parameters: Some(parameters(v, names)?),
        });
    }
    // the blob entry of each variant's own parameters, by group and place in the group's list
    let parameter_blobs: Vec<Vec<i64>> = match program.get("m_ParameterBlobIndices") {
        Some(Value::Array(Array::Values(groups))) => groups.iter().map(ints).collect::<Result<_>>()?,
        _ => Vec::new(),
    };
    for (group, list) in items(program, "m_PlayerSubPrograms")?.iter().enumerate() {
        let list: &[Value] = match list {
            Value::Array(Array::Values(x)) => x,
            Value::Array(a) if a.is_empty() => &[],
            _ => return Err(invalid("shader pass: a player sub-program list")),
        };
        for (place, v) in list.iter().enumerate() {
            let parameter_blob = parameter_blobs
                .get(group)
                .and_then(|g| g.get(place))
                .map(|i| u32_of(*i, "parameter blob index"))
                .transpose()?;
            let keywords = match v.get("m_KeywordIndices") {
                Some(k) => ints(k)?
                    .into_iter()
                    .map(|i| {
                        usize::try_from(i)
                            .ok()
                            .and_then(|i| keyword_names.get(i))
                            .cloned()
                            .ok_or_else(|| invalid(format!("shader {name}: keyword index {i}")))
                    })
                    .collect::<Result<_>>()?,
                None => Vec::new(),
            };
            variants.push(Variant {
                group,
                blob_index: u32_of(int(v, "m_BlobIndex")?, "blob index")?,
                keywords,
                gpu_program_type: int(v, "m_GpuProgramType")?,
                requirements: v.get("m_ShaderRequirements").and_then(Value::as_i64).unwrap_or(0),
                parameter_blob,
                parameters: None,
            });
        }
    }
    let parameters = match program.get("m_CommonParameters") {
        Some(c) => parameters(c, names)?,
        None => Parameters::default(),
    };
    Ok(PassProgram { variants, parameters })
}

/// Blob segments of a [`ShaderPrograms`] already decompressed ([`ShaderPrograms::code_with`]).
#[derive(Debug, Default)]
pub struct Segments(std::collections::BTreeMap<(usize, usize), std::sync::Arc<Vec<u8>>>);

impl Segments {
    fn get(
        &mut self,
        programs: &ShaderPrograms,
        platform: usize,
        k: usize,
    ) -> Result<std::sync::Arc<Vec<u8>>> {
        if let Some(s) = self.0.get(&(platform, k)) {
            return Ok(s.clone());
        }
        let s = std::sync::Arc::new(programs.segment(platform, k)?);
        self.0.insert((platform, k), s.clone());
        Ok(s)
    }
}

impl ShaderPrograms {
    /// Read the Shader `r` of `player`.
    pub fn read(player: &Player, r: &ObjRef) -> Result<Self> {
        let s = player.decode(r)?;
        let form = s
            .get("m_ParsedForm")
            .ok_or_else(|| invalid(format!("shader {}/{}: no m_ParsedForm", r.file, r.path_id)))?;
        let name = form
            .get("m_Name")
            .and_then(Value::as_str)
            .unwrap_or("")
            .to_owned();
        let keyword_names: Vec<String> = items(form, "m_KeywordNames")?
            .iter()
            .map(|k| {
                k.as_str()
                    .map(str::to_owned)
                    .ok_or_else(|| invalid("shader: a keyword name"))
            })
            .collect::<Result<_>>()?;
        let mut passes = Vec::new();
        for sub in items(form, "m_SubShaders")? {
            let mut out = Vec::new();
            for p in items(sub, "m_Passes")? {
                // the pass's name table: name -> index
                let mut names = std::collections::BTreeMap::new();
                for pair in items(p, "m_NameIndices")? {
                    let Value::Pair(kv) = pair else {
                        return Err(invalid("shader pass: m_NameIndices entry is not a pair"));
                    };
                    let key = kv.0.as_str().ok_or_else(|| invalid("shader pass: a name"))?;
                    let index =
                        kv.1.as_i64()
                            .ok_or_else(|| invalid("shader pass: a name index"))?;
                    names.insert(index, key.to_owned());
                }
                let stage = |stage: Stage| match p.get(stage.field()) {
                    Some(program) => stage_program(program, &names, &keyword_names, &name),
                    None => Ok(PassProgram::default()),
                };
                out.push(PassStages {
                    vertex: stage(Stage::Vertex)?,
                    fragment: stage(Stage::Fragment)?,
                    geometry: stage(Stage::Geometry)?,
                    hull: stage(Stage::Hull)?,
                    domain: stage(Stage::Domain)?,
                });
            }
            passes.push(out);
        }
        let platforms = s.get("platforms").map(ints).transpose()?.unwrap_or_default();
        let per = |k: &str| -> Result<Vec<Vec<i64>>> {
            items(&s, k)?
                .iter()
                .map(|v| match v {
                    Value::Array(_) => ints(v),
                    x => Ok(vec![
                        x.as_i64().ok_or_else(|| invalid(format!("shader: {k} entry")))?,
                    ]),
                })
                .collect()
        };
        let (offsets, packed, unpacked) = (
            per("offsets")?,
            per("compressedLengths")?,
            per("decompressedLengths")?,
        );
        if offsets.len() != platforms.len()
            || packed.len() != platforms.len()
            || unpacked.len() != platforms.len()
        {
            return Err(invalid(format!(
                "shader {name}: blob tables of another length than its platforms"
            )));
        }
        let segments = offsets
            .iter()
            .zip(&packed)
            .zip(&unpacked)
            .map(|((o, p), u)| {
                if o.len() != p.len() || o.len() != u.len() {
                    return Err(invalid(format!(
                        "shader {name}: segment tables of different lengths"
                    )));
                }
                let size = |n: i64| usize::try_from(n).map_err(|_| invalid("shader: a negative blob size"));
                o.iter()
                    .zip(p)
                    .zip(u)
                    .map(|((&o, &p), &u)| Ok((size(o)?, size(p)?, size(u)?)))
                    .collect()
            })
            .collect::<Result<_>>()?;
        let blob = s
            .get("compressedBlob")
            .and_then(Value::as_bytes)
            .map(<[u8]>::to_vec)
            .unwrap_or_default();
        Ok(ShaderPrograms {
            name,
            passes,
            platforms,
            segments,
            blob,
        })
    }

    /// Segment `k` of platform `platform`'s blob, decompressed.
    fn segment(&self, platform: usize, k: usize) -> Result<Vec<u8>> {
        let &(offset, packed, unpacked) = self
            .segments
            .get(platform)
            .and_then(|s| s.get(k))
            .ok_or_else(|| invalid(format!("shader {}: no blob segment {k}", self.name)))?;
        let bytes = offset
            .checked_add(packed)
            .and_then(|end| self.blob.get(offset..end))
            .ok_or_else(|| invalid(format!("shader {}: blob segment {k} past the blob", self.name)))?;
        Lz4.decompress(bytes, unpacked)
            .map_err(|e| invalid(format!("shader {}: blob segment {k}: {e}", self.name)))
    }

    /// The code of blob entry `index` of the blob for `platform` (Unity's `ShaderCompilerPlatform`): the program
    /// record's code bytes, as stored.
    pub fn code(&self, platform: i64, index: u32) -> Result<Vec<u8>> {
        self.code_with(&mut Segments::default(), platform, index)
    }

    /// [`ShaderPrograms::code`] keeping the segments it decompresses in `cache`, which reads many entries of one
    /// shader without decompressing a segment for each.
    pub fn code_with(&self, cache: &mut Segments, platform: i64, index: u32) -> Result<Vec<u8>> {
        let entry = self.entry_with(cache, platform, index)?;
        Self::code_of(&entry)
    }

    /// The bytes of blob entry `index` of the blob for `platform`, as stored.
    pub fn entry_with(&self, cache: &mut Segments, platform: i64, index: u32) -> Result<Vec<u8>> {
        let p = self
            .platforms
            .iter()
            .position(|&x| x == platform)
            .ok_or_else(|| invalid(format!("shader {}: no blob for platform {platform}", self.name)))?;
        let table = cache.get(self, p, 0)?;
        let mut r = crate::reader::Reader::new(&table, false);
        let count = usize::try_from(r.i32()?).map_err(|_| invalid("shader blob: a negative entry count"))?;
        if index as usize >= count {
            return Err(invalid(format!(
                "shader {}: blob entry {index} of {count}",
                self.name
            )));
        }
        // the table's count is its own word: the entry has to be in the bytes there are
        let at = usize::try_from(index)
            .ok()
            .and_then(|i| i.checked_mul(12))
            .and_then(|o| o.checked_add(4))
            .filter(|at| at.saturating_add(12) <= table.len())
            .ok_or_else(|| {
                invalid(format!(
                    "shader {}: blob entry {index} of {count} in a table of {} bytes",
                    self.name,
                    table.len()
                ))
            })?;
        r.pos = at;
        let (offset, length, segment) = (r.i32()?, r.i32()?, r.i32()?);
        let span = |n: i32| usize::try_from(n).map_err(|_| invalid("shader blob: a negative entry span"));
        let (offset, length, segment) = (span(offset)?, span(length)?, span(segment)?);
        let data = if segment == 0 {
            table
        } else {
            cache.get(self, p, segment)?
        };
        let entry = offset
            .checked_add(length)
            .and_then(|end| data.get(offset..end))
            .ok_or_else(|| {
                invalid(format!(
                    "shader {}: blob entry {index} past its segment",
                    self.name
                ))
            })?;
        Ok(entry.to_vec())
    }

    /// The code of a program record (`entry`): the record's version, program type, a 12-byte header, (2016.2+) 4
    /// more, keywords, code.
    fn code_of(entry: &[u8]) -> Result<Vec<u8>> {
        let mut r = crate::reader::Reader::new(entry, false);
        let version = r.i32()?;
        let _program_type = r.i32()?;
        r.take(12)?;
        if version >= 201_608_170 {
            r.take(4)?;
        }
        let strings = |r: &mut crate::reader::Reader<'_>| -> Result<()> {
            let n =
                usize::try_from(r.i32()?).map_err(|_| invalid("shader blob: a negative keyword count"))?;
            for _ in 0..n {
                let len = usize::try_from(r.i32()?)
                    .map_err(|_| invalid("shader blob: a negative string length"))?;
                r.take(len)?;
                r.align(4)?;
            }
            Ok(())
        };
        strings(&mut r)?;
        if (201_806_140..202_012_090).contains(&version) {
            strings(&mut r)?;
        }
        let len = usize::try_from(r.i32()?).map_err(|_| invalid("shader blob: a negative code length"))?;
        Ok(r.take(len)?.to_vec())
    }
}

/// Unity's `ShaderGpuProgramType` of the D3D11 programs: vertex SM 4.0 and 5.0 (15, 16), pixel (17, 18), geometry (19,
/// 20), hull (21), domain (22).
pub const D3D11_PROGRAM_TYPES: std::ops::RangeInclusive<i64> = 15..=22;

/// The DXBC container in a D3D11 program record's code: Unity stores a short header before it. `None` when the code
/// holds no container (the first `DXBC` within its first 64 bytes, whole as its own header sizes it).
pub fn dxbc(code: &[u8]) -> Option<&[u8]> {
    (0..code.len().min(64)).find_map(|at| {
        let c = code.get(at..)?;
        if c.get(..4)? != b"DXBC" {
            return None;
        }
        let size = u32::from_le_bytes(c.get(24..28)?.try_into().ok()?) as usize;
        c.get(..size)
    })
}

/// The properties a decoded shader declares (`m_ParsedForm.m_PropInfo.m_Props`); none where it states none.
fn properties(s: &Value) -> Result<&[Value]> {
    Ok(s.get("m_ParsedForm")
        .and_then(|f| f.get("m_PropInfo"))
        .map(|p| items(p, "m_Props"))
        .transpose()?
        .unwrap_or(&[]))
}

/// The default each texture property of shader `r` declares (`m_ParsedForm.m_PropInfo.m_Props`, type 4:
/// `m_DefTexture.m_DefaultName`), by property.
pub fn texture_defaults(player: &Player, r: &ObjRef) -> Result<Vec<(String, String)>> {
    let s = player.decode(r)?;
    let props = properties(&s)?;
    let mut out = Vec::new();
    for p in props {
        if p.get("m_Type").and_then(Value::as_i64) != Some(4) {
            continue;
        }
        let name = p.get("m_Name").and_then(Value::as_str);
        let default = p
            .get("m_DefTexture")
            .and_then(|d| d.get("m_DefaultName"))
            .and_then(Value::as_str);
        if let (Some(name), Some(default)) = (name, default) {
            out.push((name.to_owned(), default.to_owned()));
        }
    }
    Ok(out)
}

/// Unity's `ShaderPropertyFlags.HDR`: a colour property declared `[HDR]`.
pub const PROPERTY_HDR: u32 = 16;

/// A colour property a shader declares (`m_ParsedForm.m_PropInfo.m_Props`, type 0) and its `m_Flags` (Unity's
/// `ShaderPropertyFlags`).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ColourProperty {
    /// The property's name (`_BaseColor`).
    pub name: String,
    /// Its `m_Flags` (`ShaderPropertyFlags`; [`PROPERTY_HDR`] is `[HDR]`).
    pub flags: u32,
}

impl ColourProperty {
    /// Whether the property is declared `[HDR]` ([`PROPERTY_HDR`]). In a linear colour space player Unity hands an HDR
    /// colour's stored value to the programs as it is, and converts any other colour from gamma to linear.
    pub fn hdr(&self) -> bool {
        self.flags & PROPERTY_HDR != 0
    }
}

/// The colour properties shader `r` declares, with their flags, in its order.
pub fn colour_properties(player: &Player, r: &ObjRef) -> Result<Vec<ColourProperty>> {
    let s = player.decode(r)?;
    let mut out = Vec::new();
    for p in properties(&s)? {
        if p.get("m_Type").and_then(Value::as_i64) != Some(0) {
            continue;
        }
        let name = p
            .get("m_Name")
            .and_then(Value::as_str)
            .ok_or_else(|| invalid("shader property: a colour property has no m_Name"))?;
        let flags = p
            .get("m_Flags")
            .and_then(Value::as_i64)
            .and_then(|f| u32::try_from(f).ok())
            .ok_or_else(|| invalid(format!("shader property {name}: m_Flags is not a u32")))?;
        out.push(ColourProperty {
            name: name.to_owned(),
            flags,
        });
    }
    Ok(out)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn word(out: &mut Vec<u8>, w: u32) {
        out.extend_from_slice(&w.to_le_bytes());
    }

    fn text(out: &mut Vec<u8>, s: &str) {
        word(out, s.len() as u32);
        out.extend_from_slice(s.as_bytes());
        while !out.len().is_multiple_of(4) {
            out.push(0);
        }
    }

    /// A blob entry as a 2021.2 player lays it out: two buffers (`$Globals` with a vector and a matrix, and
    /// `UnityPerMaterial` with none), two textures, the binding of `UnityPerMaterial` to slot 6, and a sampler.
    fn blob() -> Vec<u8> {
        let mut b = Vec::new();
        for w in [PARAMETER_BLOB_MAGIC, 3, 0, 0, 0, 0] {
            word(&mut b, w);
        }
        text(&mut b, "$Globals");
        word(&mut b, 128);
        word(&mut b, 2);
        text(&mut b, "_Color");
        for w in [0, 1, 4, 0, 0, 16] {
            word(&mut b, w);
        }
        text(&mut b, "unity_MatrixV");
        for w in [0, 4, 4, 1, 0, 64] {
            word(&mut b, w);
        }
        word(&mut b, 0);
        text(&mut b, "UnityPerMaterial");
        for w in [208, 0, 0] {
            word(&mut b, w);
        }
        word(&mut b, 4);
        text(&mut b, "_IceTex");
        for w in [0, 0, 4, 4] {
            word(&mut b, w);
        }
        text(&mut b, "_Roughness");
        for w in [0, 7, u32::MAX, 4] {
            word(&mut b, w);
        }
        text(&mut b, "UnityPerMaterial");
        for w in [1, 6, 0] {
            word(&mut b, w);
        }
        text(&mut b, "");
        for w in [4, 3, 341] {
            word(&mut b, w);
        }
        b
    }

    fn four_wide(name: &str, offset: u32) -> Field {
        Field {
            name: name.into(),
            offset,
            kind: 0,
            shape: Shape::Vector { columns: 4 },
            array_size: 0,
        }
    }

    #[test]
    fn a_parameter_blob_gives_buffers_slots_textures_and_samplers() {
        let p = parameter_blob(&blob()).unwrap();
        assert_eq!(p.constant_buffers.len(), 2);
        let globals = &p.constant_buffers[0];
        assert_eq!(
            (globals.name.as_str(), globals.size, globals.slot),
            ("$Globals", 128, None)
        );
        assert_eq!(
            globals.fields,
            [
                four_wide("_Color", 16),
                Field {
                    name: "unity_MatrixV".into(),
                    offset: 64,
                    kind: 0,
                    shape: Shape::Matrix { rows: 4 },
                    array_size: 0
                },
            ]
        );
        assert_eq!(p.constant_buffers[1].slot, Some(6));
        assert_eq!(
            p.textures,
            [
                TextureParam {
                    name: "_IceTex".into(),
                    slot: 0,
                    sampler: Some(4),
                    dim: 4
                },
                TextureParam {
                    name: "_Roughness".into(),
                    slot: 7,
                    sampler: None,
                    dim: 4
                },
            ]
        );
        assert_eq!(p.samplers, [(341, 3)]);
    }

    #[test]
    fn a_parameter_blob_that_is_not_the_format_or_holds_what_is_not_read_is_refused() {
        let mut other = blob();
        other[0] ^= 1;
        assert!(parameter_blob(&other).is_err());
        // bytes after the last entry
        let mut long = blob();
        long.extend_from_slice(&[0; 4]);
        assert!(parameter_blob(&long).is_err());
        // a kind 2 entry (a structured buffer) is not read: the sampler entry's last three words become kind 2
        let mut structured = blob();
        let at = structured.len() - 12;
        structured[at..at + 4].copy_from_slice(&2u32.to_le_bytes());
        assert!(parameter_blob(&structured).is_err());
        // cut short
        assert!(parameter_blob(&blob()[..40]).is_err());
    }

    #[test]
    fn a_variants_parameters_are_its_blobs_with_the_pass_common_ones_added() {
        let own = parameter_blob(&blob()).unwrap();
        let common = Parameters {
            constant_buffers: vec![
                ConstantBuffer {
                    name: "$Globals".into(),
                    slot: Some(0),
                    size: 256,
                    fields: vec![four_wide("_Color", 16), four_wide("_Other", 32)],
                    structs: Vec::new(),
                },
                ConstantBuffer {
                    name: "AdditionalLights".into(),
                    slot: Some(1),
                    size: 24_576,
                    fields: vec![four_wide("_Lights", 0)],
                    structs: Vec::new(),
                },
            ],
            textures: vec![TextureParam {
                name: "_Common".into(),
                slot: 0,
                sampler: None,
                dim: 4,
            }],
            samplers: vec![(1, 0), (341, 3)],
        };
        let merged = merge_parameters(&common, &own);
        let names: Vec<&str> = merged.constant_buffers.iter().map(|c| c.name.as_str()).collect();
        assert_eq!(names, ["$Globals", "UnityPerMaterial", "AdditionalLights"]);
        let globals = &merged.constant_buffers[0];
        // the blob's fields and the common one it lacks, in offset order; the larger size; the common slot
        let fields: Vec<(&str, u32)> = globals
            .fields
            .iter()
            .map(|f| (f.name.as_str(), f.offset))
            .collect();
        assert_eq!(fields, [("_Color", 16), ("_Other", 32), ("unity_MatrixV", 64)]);
        assert_eq!((globals.size, globals.slot), (256, Some(0)));
        assert_eq!(merged.constant_buffers[1].slot, Some(6));
        // a common texture in the slot the blob names is the blob's: it is not kept twice
        assert_eq!(merged.textures.len(), 2);
        assert_eq!(merged.textures[0].name, "_IceTex");
        assert_eq!(merged.samplers, [(341, 3), (1, 0)]);
    }

    /// A blob table that states more entries than it holds: the entry is refused, not indexed past the table.
    #[test]
    fn a_blob_entry_past_the_table_is_refused() {
        // one LZ4 literal run of the four bytes of a count of 1000
        let table = vec![0x40u8, 0xE8, 0x03, 0, 0];
        let programs = ShaderPrograms {
            name: "s".into(),
            passes: vec![],
            platforms: vec![4],
            segments: vec![vec![(0, table.len(), 4)]],
            blob: table,
        };
        let e = programs.code(4, 999).unwrap_err().to_string();
        assert!(e.contains("blob entry 999 of 1000 in a table of 4 bytes"), "{e}");
    }
    use std::sync::Arc;

    #[test]
    fn the_dxbc_container_is_found_past_unitys_header() {
        let mut container = b"DXBC".to_vec();
        container.extend([0; 20]);
        container.extend(32u32.to_le_bytes());
        container.extend([0; 4]);
        let mut code = vec![7, 0, 0, 0, 1, 2];
        code.extend(&container);
        code.extend([9, 9]);
        assert_eq!(dxbc(&code), Some(&container[..]));
        assert_eq!(dxbc(&code[..20]), None);
    }

    fn obj(fields: Vec<(&str, Value)>) -> Value {
        Value::Struct(fields.into_iter().map(|(k, v)| (Arc::from(k), v)).collect())
    }

    fn list(items: Vec<Value>) -> Value {
        Value::Array(Array::Values(items))
    }

    fn vector(name: i64, offset: i64, dim: i64) -> Value {
        obj(vec![
            ("m_NameIndex", Value::Int(name)),
            ("m_Index", Value::Int(offset)),
            ("m_ArraySize", Value::Int(0)),
            ("m_Type", Value::Int(0)),
            ("m_Dim", Value::Int(dim)),
        ])
    }

    fn cbuffer(fields: Vec<Value>, structs: Vec<Value>) -> Value {
        obj(vec![
            ("m_NameIndex", Value::Int(0)),
            ("m_VectorParams", list(fields)),
            ("m_MatrixParams", list(Vec::new())),
            ("m_StructParams", list(structs)),
            ("m_Size", Value::Int(32)),
        ])
    }

    fn params(cbs: Vec<Value>) -> Value {
        obj(vec![
            ("m_ConstantBuffers", list(cbs)),
            (
                "m_ConstantBufferBindings",
                list(vec![obj(vec![
                    ("m_NameIndex", Value::Int(0)),
                    ("m_Index", Value::Int(3)),
                ])]),
            ),
        ])
    }

    #[test]
    fn parameters_resolve_names_slots_offsets_and_structs_and_refuse_what_they_cannot() {
        let names: std::collections::BTreeMap<i64, String> = [
            (0, "UnityPerMaterial".to_owned()),
            (1, "_BaseColor".to_owned()),
            (2, "_Smoothness".to_owned()),
        ]
        .into_iter()
        .collect();
        // fields come out by offset, the buffer at the slot its binding names
        let p = parameters(
            &params(vec![cbuffer(vec![vector(2, 16, 1), vector(1, 0, 4)], Vec::new())]),
            &names,
        )
        .unwrap();
        let cb = &p.constant_buffers[0];
        assert_eq!(
            (cb.name.as_str(), cb.slot, cb.size),
            ("UnityPerMaterial", Some(3), 32)
        );
        assert_eq!(
            cb.fields
                .iter()
                .map(|f| (f.name.as_str(), f.offset, f.shape))
                .collect::<Vec<_>>(),
            vec![
                ("_BaseColor", 0, Shape::Vector { columns: 4 }),
                ("_Smoothness", 16, Shape::Vector { columns: 1 })
            ]
        );
        // a struct field: an array of members (instancing's per-draw arrays)
        let instanced = obj(vec![
            ("m_NameIndex", Value::Int(1)),
            ("m_Index", Value::Int(0)),
            ("m_ArraySize", Value::Int(2)),
            ("m_StructSize", Value::Int(32)),
            ("m_VectorMembers", list(vec![vector(2, 16, 4)])),
            ("m_MatrixMembers", list(Vec::new())),
        ]);
        let p = parameters(&params(vec![cbuffer(Vec::new(), vec![instanced])]), &names).unwrap();
        let s = &p.constant_buffers[0].structs[0];
        assert_eq!(
            (s.name.as_str(), s.offset, s.array_size, s.size),
            ("_BaseColor", 0, 2, 32)
        );
        assert_eq!(
            s.members
                .iter()
                .map(|f| (f.name.as_str(), f.offset))
                .collect::<Vec<_>>(),
            vec![("_Smoothness", 16)]
        );
        // a name index past the table and a negative offset are refused
        assert!(parameters(&params(vec![cbuffer(vec![vector(9, 0, 4)], Vec::new())]), &names).is_err());
        assert!(parameters(&params(vec![cbuffer(vec![vector(1, -4, 4)], Vec::new())]), &names).is_err());
    }

    #[test]
    fn integer_arrays_read_whatever_their_element_type() {
        assert_eq!(ints(&Value::Array(Array::U16(vec![1, 2]))).unwrap(), vec![1, 2]);
        assert_eq!(
            ints(&list(vec![Value::Int(-1), Value::UInt(5)])).unwrap(),
            vec![-1, 5]
        );
        assert!(ints(&Value::Array(Array::F32(vec![1.0]))).is_err());
        assert!(ints(&Value::Int(1)).is_err());
    }
}