Skip to main content

zenpixels_convert/
convert.rs

1//! Row-level pixel conversion kernels.
2//!
3//! Each kernel converts one row of `width` pixels from a source format to
4//! a destination format. Individual step kernels are pure functions with
5//! no allocation. Multi-step plans use [`ConvertScratch`] ping-pong
6//! buffers to avoid per-row heap allocation in streaming loops.
7
8use alloc::vec;
9use alloc::vec::Vec;
10use core::cmp::min;
11
12use crate::policy::{AlphaPolicy, ConvertOptions, DepthPolicy, LumaCoefficients};
13use crate::{
14    AlphaMode, ChannelLayout, ChannelType, ColorPrimaries, ConvertError, PixelDescriptor,
15    TransferFunction,
16};
17use whereat::{At, ResultAtExt};
18
19/// HDR→SDR tone-mapping configuration for
20/// [`ConvertPlan::new_with_hdr_config`].
21///
22/// Bundles the source-peak luminance (mandatory — the curve is
23/// parameterized by it), target-peak luminance (typically 100 cd/m² for
24/// SDR), and the OKLch soft chroma-compression knee (production default `0.96`).
25///
26/// Construct via [`for_source_peak`](Self::for_source_peak) and refine
27/// with the `with_*` builders — the struct is `#[non_exhaustive]` so
28/// future knobs can land additively without a breaking release. The
29/// existing fields stay `pub` for reading (and in-place mutation).
30///
31/// Both peak fields must be **positive and finite**;
32/// [`ConvertPlan::new_with_hdr_config`] rejects anything else (including
33/// the unset `Default` value) with
34/// [`ConvertError::HdrSourceRequiresPeak`](crate::ConvertError::HdrSourceRequiresPeak)
35/// instead of silently tone-mapping through a degenerate curve.
36#[cfg(feature = "hdr-experimental")]
37#[derive(Clone, Copy, Debug, PartialEq)]
38#[non_exhaustive]
39pub struct HdrConfig {
40    /// HDR source peak luminance in cd/m². The BT.2446-A curve treats
41    /// `1.0` source-normalized as this peak. Typical values: 1000 (HDR10,
42    /// Apple HDR), 4000 (HDR10+ reference), 10000 (PQ peak).
43    pub source_peak_nits: f32,
44    /// SDR target peak luminance in cd/m². Typical value: 100 (BT.709 /
45    /// sRGB diffuse-white peak; BT.1886 reference).
46    pub target_peak_nits: f32,
47    /// Fraction of max chroma where OKLch soft compression kicks in
48    /// (`0.0`–`1.0`). `0.96` (the default) compresses only the outermost
49    /// 4 % of the gamut; lower values bring the compression in earlier.
50    /// Ignored when the target primaries are BT.2020 (wide-gamut output
51    /// mode emits no `SoftCompressOklch` step).
52    ///
53    /// The default was calibrated against the 76-sample imazen-26
54    /// gain-mapped corpus on 2026-06-23: `0.96` is the largest knee
55    /// (least desaturation) where the corpus-p90 fraction of pre-clamp
56    /// out-of-gamut pixels stays under 0.1 %. Findings:
57    /// [`zentone/benchmarks/softcompress_knee_findings_2026-06-23.md`](https://github.com/imazen/zentone).
58    pub gamut_knee: f32,
59}
60
61#[cfg(feature = "hdr-experimental")]
62impl Default for HdrConfig {
63    /// Pipeline defaults: `target_peak_nits = 100.0` (SDR reference white)
64    /// and `gamut_knee = 0.96` (empirically calibrated against the
65    /// imazen-26 gain-mapped HDR corpus, 2026-06-23). `source_peak_nits`
66    /// has no default — it returns `0.0` ("unset"), which
67    /// [`ConvertPlan::new_with_hdr_config`] rejects with
68    /// [`ConvertError::HdrSourceRequiresPeak`](crate::ConvertError::HdrSourceRequiresPeak).
69    /// Start from [`HdrConfig::for_source_peak`] instead.
70    fn default() -> Self {
71        Self {
72            source_peak_nits: 0.0,
73            target_peak_nits: 100.0,
74            gamut_knee: 0.96,
75        }
76    }
77}
78
79#[cfg(feature = "hdr-experimental")]
80impl HdrConfig {
81    /// Config for an HDR source with the given peak luminance in cd/m²,
82    /// keeping the calibrated defaults for everything else
83    /// (`target_peak_nits = 100.0`, `gamut_knee = 0.96`).
84    ///
85    /// Typical `source_peak_nits` values: 1000 (HDR10, Apple HDR), 4000
86    /// (HDR10+ reference), 10000 (PQ peak) — or better, the measured
87    /// MaxCLL of the actual content
88    /// ([`CllMeasure::measure_max`](crate::hdr::measure::CllMeasure::measure_max)).
89    #[must_use]
90    pub fn for_source_peak(source_peak_nits: f32) -> Self {
91        Self {
92            source_peak_nits,
93            ..Self::default()
94        }
95    }
96
97    /// Set the SDR target peak luminance in cd/m² (default `100.0`).
98    #[must_use]
99    pub fn with_target_peak_nits(mut self, nits: f32) -> Self {
100        self.target_peak_nits = nits;
101        self
102    }
103
104    /// Set the OKLch soft-compression knee (default `0.96`; meaningful
105    /// range `0.0..=1.0` — values outside it are not validated and
106    /// produce under-/over-compression rather than an error).
107    #[must_use]
108    pub fn with_gamut_knee(mut self, knee: f32) -> Self {
109        self.gamut_knee = knee;
110        self
111    }
112}
113
114/// True when `(from.transfer, to.transfer)` describes an HDR→SDR
115/// transition that requires the BT.2446-A tone map step.
116///
117/// PQ / HLG source to an SDR-encoded target (`Srgb` / `Bt709` /
118/// `Gamma22`). The `Linear` target is **not** considered SDR here —
119/// decoding a PQ buffer to relative-linear F32 preserves the data
120/// losslessly (the value is just in a different transfer-function
121/// representation), and the caller may downstream apply their own
122/// tone mapping or carry the wide dynamic range through. HLG↔PQ is
123/// handled by the dedicated refusal upstream (different luminance
124/// domains, no straight tone-map path).
125#[cfg(feature = "hdr-experimental")]
126fn is_hdr_to_sdr(from: TransferFunction, to: TransferFunction) -> bool {
127    let src_is_hdr = matches!(from, TransferFunction::Pq | TransferFunction::Hlg);
128    let dst_is_sdr_encoded = matches!(
129        to,
130        TransferFunction::Srgb | TransferFunction::Bt709 | TransferFunction::Gamma22
131    );
132    src_is_hdr && dst_is_sdr_encoded
133}
134
135/// Pre-computed conversion plan.
136///
137/// Stores the chain of steps needed to convert from one format to another.
138/// Created once, applied to every row.
139#[derive(Clone, Debug)]
140pub struct ConvertPlan {
141    pub(crate) from: PixelDescriptor,
142    pub(crate) to: PixelDescriptor,
143    pub(crate) steps: Vec<ConvertStep>,
144    /// Relative-linear → PQ-absolute scale = `diffuse_white_nits / 10000`,
145    /// applied by the PQ kernels (encode multiplies pre-OETF, decode divides
146    /// post-EOTF). `1.0` is the unsignaled default and means "treat linear as
147    /// already PQ-absolute (1.0 = 10000 cd/m²)" — i.e. exactly the prior
148    /// behavior, so plans built without an anchor are byte-for-byte unchanged.
149    /// Set via [`with_pq_anchor`](Self::with_pq_anchor). HLG steps ignore it
150    /// (scene-referred — different anchoring, out of scope here).
151    pub(crate) pq_anchor_scale: f32,
152}
153
154/// Selects which fused TF + matrix + TF kernel a [`ConvertStep::Fused`]
155/// dispatches to. Each variant is one (source-TF, source-depth, dest-depth,
156/// dest-TF, channel-shape) shape that the planner can peephole.
157#[derive(Clone, Copy, Debug, PartialEq, Eq)]
158pub(crate) enum FusedKind {
159    /// `SrgbU8 → matrix → SrgbU8`, 3-channel RGB.
160    SrgbU8GamutRgb,
161    /// `SrgbU8 → matrix → SrgbU8`, 4-channel RGBA (alpha passthrough).
162    SrgbU8GamutRgba,
163    /// `SrgbU16 → matrix → SrgbU16`, 3-channel RGB via 65K-entry LUTs.
164    SrgbU16GamutRgb,
165    /// `SrgbU8 → matrix → LinearF32`, 3-channel RGB (cross-depth);
166    /// output preserves extended range (no clamp).
167    SrgbU8ToLinearF32Rgb,
168    /// `LinearF32 → matrix → SrgbU8`, 3-channel RGB (cross-depth);
169    /// always clamps since u8 can't represent out-of-gamut values.
170    LinearF32ToSrgbU8Rgb,
171}
172
173impl FusedKind {
174    /// The historical per-variant name kept stable for the
175    /// `__trace_ops` recorder + `tests/plan_validation.rs` (which still
176    /// asserts on `s.contains("FusedSrgb")`).
177    #[inline]
178    #[allow(dead_code)] // used only when `__trace_ops` feature is enabled
179    pub(crate) const fn variant_name(self) -> &'static str {
180        match self {
181            Self::SrgbU8GamutRgb => "FusedSrgbU8GamutRgb",
182            Self::SrgbU8GamutRgba => "FusedSrgbU8GamutRgba",
183            Self::SrgbU16GamutRgb => "FusedSrgbU16GamutRgb",
184            Self::SrgbU8ToLinearF32Rgb => "FusedSrgbU8ToLinearF32Rgb",
185            Self::LinearF32ToSrgbU8Rgb => "FusedLinearF32ToSrgbU8Rgb",
186        }
187    }
188}
189
190/// A single conversion step.
191///
192/// Not `Copy` — some variants (e.g., `ExternalTransform`) carry an
193/// `Arc`. Peephole rewrites must use `.clone()` or index assignment with
194/// pattern matching instead of `*step` dereferences.
195#[derive(Clone, Debug)]
196pub(crate) enum ConvertStep {
197    /// No-op (identity).
198    Identity,
199    /// BGRA → RGBA byte swizzle (or vice versa).
200    SwizzleBgraRgba,
201    /// Fused RGB → BGRA: byte swap + add opaque alpha in a single SIMD pass.
202    /// Equivalent to `[AddAlpha, SwizzleBgraRgba]` but writes the destination
203    /// once instead of twice.
204    RgbToBgra,
205    /// Add alpha channel (3ch → 4ch), filling with opaque.
206    AddAlpha,
207    /// Drop alpha channel (4ch → 3ch).
208    DropAlpha,
209    /// Composite onto solid matte color, then drop alpha (4ch → 3ch).
210    ///
211    /// Blends in linear light using the source descriptor's transfer
212    /// function: pixel RGB is EOTF'd per source TF, alpha-blended against
213    /// the pre-linearized matte, then OETF'd back to source TF. Alpha is
214    /// treated as linear regardless of color-channel TF. The matte
215    /// `(r, g, b)` is always interpreted as sRGB u8 (CSS-style background).
216    ///
217    /// Implemented uniformly across U8/U16/F32/F16 via per-TF
218    /// monomorphization; sRGB integer paths use LUT-based EOTF/OETF.
219    MatteComposite { r: u8, g: u8, b: u8 },
220    /// Gray → RGB (replicate gray to all 3 channels).
221    GrayToRgb,
222    /// Gray → RGBA (replicate + opaque alpha).
223    GrayToRgba,
224    /// RGB → Gray (Y' encoded luma — coefficients applied to encoded bytes).
225    ///
226    /// The semantic is BT.709/BT.601/etc. Y' (encoded luma), NOT linear-light
227    /// luminance L. This is fast, exactly round-trips for `R==G==B` inputs,
228    /// and matches what JPEG/video pipelines compute. Linear-light luminance
229    /// would require linearize → weight → encode and is not currently
230    /// surfaced; document any future linear-L pathway as a separate variant.
231    ///
232    /// Coefficients are resolved from `ConvertOptions::luma` at plan build
233    /// time (`new_explicit`). Default for plans built via `Self::new`
234    /// without options is `LumaCoefficients::Bt709`.
235    RgbToGray { coefficients: LumaCoefficients },
236    /// RGBA → Gray, drop alpha. See [`RgbToGray`](Self::RgbToGray) for
237    /// semantic and coefficient resolution.
238    RgbaToGray { coefficients: LumaCoefficients },
239    /// GrayAlpha → RGBA (replicate gray, keep alpha).
240    GrayAlphaToRgba,
241    /// GrayAlpha → RGB (replicate gray, drop alpha).
242    GrayAlphaToRgb,
243    /// Gray → GrayAlpha (add opaque alpha).
244    GrayToGrayAlpha,
245    /// GrayAlpha → Gray (drop alpha).
246    GrayAlphaToGray,
247    /// sRGB u8 → linear f32 (per channel, EOTF).
248    SrgbU8ToLinearF32,
249    /// Linear f32 → sRGB u8 (per channel, OETF).
250    LinearF32ToSrgbU8,
251    /// Naive u8 → f32 (v / 255.0, no gamma).
252    NaiveU8ToF32,
253    /// Naive f32 → u8 (clamp * 255 + 0.5, no gamma).
254    NaiveF32ToU8,
255    /// u16 → u8 ((v * 255 + 32768) >> 16).
256    U16ToU8,
257    /// u8 → u16 (v * 257).
258    U8ToU16,
259    /// u16 → f32 (v / 65535.0).
260    U16ToF32,
261    /// f32 → u16 (clamp * 65535 + 0.5).
262    F32ToU16,
263    /// f16 → f32 (IEEE 754 half-precision unpack, no TF).
264    F16ToF32,
265    /// f32 → f16 (round-to-nearest-even, no TF).
266    F32ToF16,
267    /// PQ (SMPTE ST 2084) u16 → linear f32 (EOTF).
268    PqU16ToLinearF32,
269    /// Linear f32 → PQ u16 (inverse EOTF / OETF).
270    LinearF32ToPqU16,
271    /// PQ f32 `[0,1]` → linear f32 (EOTF, no depth change).
272    PqF32ToLinearF32,
273    /// Linear f32 → PQ f32 `[0,1]` (OETF, no depth change).
274    LinearF32ToPqF32,
275    /// HLG (ARIB STD-B67) u16 → linear f32 (EOTF).
276    HlgU16ToLinearF32,
277    /// Linear f32 → HLG u16 (OETF).
278    LinearF32ToHlgU16,
279    /// HLG f32 `[0,1]` → linear f32 (EOTF, no depth change).
280    HlgF32ToLinearF32,
281    /// Linear f32 → HLG f32 `[0,1]` (OETF, no depth change).
282    LinearF32ToHlgF32,
283    /// sRGB f32 `[0,1]` → linear f32 (EOTF, no depth change). Clamps input.
284    SrgbF32ToLinearF32,
285    /// Linear f32 → sRGB f32 `[0,1]` (OETF, no depth change). Clamps output.
286    LinearF32ToSrgbF32,
287    /// sRGB f32 → linear f32 (EOTF, sign-preserving extended range).
288    /// Emitted when `ConvertOptions::clip_out_of_gamut == false`.
289    SrgbF32ToLinearF32Extended,
290    /// Linear f32 → sRGB f32 (OETF, sign-preserving extended range).
291    LinearF32ToSrgbF32Extended,
292    /// BT.709 f32 `[0,1]` → linear f32 (EOTF, no depth change).
293    Bt709F32ToLinearF32,
294    /// Linear f32 → BT.709 f32 `[0,1]` (OETF, no depth change).
295    LinearF32ToBt709F32,
296    /// Gamma 2.2 (Adobe RGB 1998) f32 `[0,1]` → linear f32 (EOTF, no depth change).
297    /// Uses the Adobe RGB 1998 canonical exponent 563/256 ≈ 2.19921875.
298    Gamma22F32ToLinearF32,
299    /// Linear f32 → Gamma 2.2 (Adobe RGB 1998) f32 `[0,1]` (OETF, no depth change).
300    LinearF32ToGamma22F32,
301    /// Straight → Premultiplied alpha.
302    StraightToPremul,
303    /// Premultiplied → Straight alpha.
304    PremulToStraight,
305    /// Linear RGB f32 → Oklab f32 (3-channel color model change).
306    LinearRgbToOklab,
307    /// Oklab f32 → Linear RGB f32 (3-channel color model change).
308    OklabToLinearRgb,
309    /// Linear RGBA f32 → Oklaba f32 (4-channel, alpha preserved).
310    LinearRgbaToOklaba,
311    /// Oklaba f32 → Linear RGBA f32 (4-channel, alpha preserved).
312    OklabaToLinearRgba,
313    /// Apply a 3×3 gamut matrix to linear RGB f32 (3 channels per pixel).
314    ///
315    /// Used for color primaries conversion (e.g., BT.709 ↔ Display P3 ↔ BT.2020).
316    /// Data must be in linear light. The matrix is row-major `[[f32; 3]; 3]`
317    /// flattened to `[f32; 9]`.
318    GamutMatrixRgbF32([f32; 9]),
319    /// Apply a 3×3 gamut matrix to linear RGBA f32 (4 channels, alpha passthrough).
320    GamutMatrixRgbaF32([f32; 9]),
321    /// Fused TF + 3×3 gamut + TF in one pass. Carries the matrix flattened
322    /// row-major to `[f32; 9]`, plus a [`FusedKind`] tag selecting which
323    /// linearize → matrix → encode shape to dispatch. Replaces the 3-step
324    /// sequence `[<lin>, GamutMatrix*F32, <enc>]` whenever the planner can
325    /// peephole it. See [`FusedKind`] for the supported shapes.
326    Fused { kind: FusedKind, matrix: [f32; 9] },
327    /// BT.2446 Method A HDR→SDR tone-map on linear-light f32 RGB in BT.2020
328    /// primaries. The plan builder ensures this step sees BT.2020 linear-light
329    /// input via preceding gamut-matrix steps; a following gamut-matrix step
330    /// (BT.2020 → target.primaries) handles the destination primaries.
331    /// Input is source-normalized (`1.0 = source_peak_nits`); output is
332    /// target-normalized (`1.0 = target_peak_nits`). RGB-only — alpha is
333    /// handled at descriptor-layout level (the planner pairs this with the
334    /// appropriate RGB/RGBA carrier).
335    ///
336    /// Gated behind `hdr-experimental` at the kernel side.
337    #[cfg(feature = "hdr-experimental")]
338    ToneMapBt2446A {
339        source_peak_nits: f32,
340        target_peak_nits: f32,
341    },
342    /// OKLch soft chroma compression on linear-light f32 RGB in
343    /// `primaries`. Pulls residual out-of-gamut excursions back into the
344    /// target unit cube using a hue-preserving rational knee curve.
345    /// Skipped (no step emitted) when the target is BT.2020 — the
346    /// wide-gamut output mode.
347    ///
348    /// Gated behind `hdr-experimental` at the kernel side.
349    #[cfg(feature = "hdr-experimental")]
350    SoftCompressOklch {
351        primaries: ColorPrimaries,
352        knee: f32,
353    },
354}
355
356impl ConvertStep {
357    /// The stable variant name used by the `__trace_ops` recorder. Kept as
358    /// a `const fn` on `ConvertStep` (no `strum`/proc-macro dep) so the
359    /// recorder has one source of truth — historically there were several
360    /// independent 60-arm matches that drifted easily.
361    #[inline]
362    #[allow(dead_code)] // used only when `__trace_ops` feature is enabled
363    pub(crate) const fn variant_name(&self) -> &'static str {
364        match self {
365            Self::Identity => "Identity",
366            Self::SwizzleBgraRgba => "SwizzleBgraRgba",
367            Self::RgbToBgra => "RgbToBgra",
368            Self::AddAlpha => "AddAlpha",
369            Self::DropAlpha => "DropAlpha",
370            Self::MatteComposite { .. } => "MatteComposite",
371            Self::GrayToRgb => "GrayToRgb",
372            Self::GrayToRgba => "GrayToRgba",
373            Self::RgbToGray { .. } => "RgbToGray",
374            Self::RgbaToGray { .. } => "RgbaToGray",
375            Self::GrayAlphaToRgba => "GrayAlphaToRgba",
376            Self::GrayAlphaToRgb => "GrayAlphaToRgb",
377            Self::GrayToGrayAlpha => "GrayToGrayAlpha",
378            Self::GrayAlphaToGray => "GrayAlphaToGray",
379            Self::SrgbU8ToLinearF32 => "SrgbU8ToLinearF32",
380            Self::LinearF32ToSrgbU8 => "LinearF32ToSrgbU8",
381            Self::NaiveU8ToF32 => "NaiveU8ToF32",
382            Self::NaiveF32ToU8 => "NaiveF32ToU8",
383            Self::U16ToU8 => "U16ToU8",
384            Self::U8ToU16 => "U8ToU16",
385            Self::U16ToF32 => "U16ToF32",
386            Self::F32ToU16 => "F32ToU16",
387            Self::F16ToF32 => "F16ToF32",
388            Self::F32ToF16 => "F32ToF16",
389            Self::PqU16ToLinearF32 => "PqU16ToLinearF32",
390            Self::LinearF32ToPqU16 => "LinearF32ToPqU16",
391            Self::PqF32ToLinearF32 => "PqF32ToLinearF32",
392            Self::LinearF32ToPqF32 => "LinearF32ToPqF32",
393            Self::HlgU16ToLinearF32 => "HlgU16ToLinearF32",
394            Self::LinearF32ToHlgU16 => "LinearF32ToHlgU16",
395            Self::HlgF32ToLinearF32 => "HlgF32ToLinearF32",
396            Self::LinearF32ToHlgF32 => "LinearF32ToHlgF32",
397            Self::SrgbF32ToLinearF32 => "SrgbF32ToLinearF32",
398            Self::LinearF32ToSrgbF32 => "LinearF32ToSrgbF32",
399            Self::SrgbF32ToLinearF32Extended => "SrgbF32ToLinearF32Extended",
400            Self::LinearF32ToSrgbF32Extended => "LinearF32ToSrgbF32Extended",
401            Self::Bt709F32ToLinearF32 => "Bt709F32ToLinearF32",
402            Self::LinearF32ToBt709F32 => "LinearF32ToBt709F32",
403            Self::Gamma22F32ToLinearF32 => "Gamma22F32ToLinearF32",
404            Self::LinearF32ToGamma22F32 => "LinearF32ToGamma22F32",
405            Self::StraightToPremul => "StraightToPremul",
406            Self::PremulToStraight => "PremulToStraight",
407            Self::LinearRgbToOklab => "LinearRgbToOklab",
408            Self::OklabToLinearRgb => "OklabToLinearRgb",
409            Self::LinearRgbaToOklaba => "LinearRgbaToOklaba",
410            Self::OklabaToLinearRgba => "OklabaToLinearRgba",
411            Self::GamutMatrixRgbF32(_) => "GamutMatrixRgbF32",
412            Self::GamutMatrixRgbaF32(_) => "GamutMatrixRgbaF32",
413            Self::Fused { kind, .. } => kind.variant_name(),
414            #[cfg(feature = "hdr-experimental")]
415            Self::ToneMapBt2446A { .. } => "ToneMapBt2446A",
416            #[cfg(feature = "hdr-experimental")]
417            Self::SoftCompressOklch { .. } => "SoftCompressOklch",
418        }
419    }
420}
421
422/// Color models that zenpixels-convert's built-in kernels resolve natively.
423///
424/// Anything outside this set is a device-dependent / CMS-only path —
425/// CMYK today, Lab / XYZ / spot inks if/when those land as
426/// [`crate::ColorModel`] variants. See [`requires_cms`].
427#[inline]
428fn native_color_model(m: crate::ColorModel) -> bool {
429    // `Gray`, `Rgb` and `Oklab` are the colorimetric spaces the built-in
430    // kernels handle end-to-end (gamut matrices, transfer LUTs, fused
431    // matluts, polyfit decoders, the OKLab gamut-compression path, …).
432    // `YCbCr` is also colorimetric-equivalent to RGB once the matrix has
433    // been applied, but no kernel here consumes raw `YCbCr` pixels: every
434    // entry point that touches YCbCr first lifts it into RGB via the
435    // decoder's own coefficient pair, so the planner never sees a
436    // `YCbCr` color model on either side. CMYK is the only non-native
437    // model that currently reaches the planner.
438    matches!(
439        m,
440        crate::ColorModel::Gray | crate::ColorModel::Rgb | crate::ColorModel::Oklab
441    )
442}
443
444/// True when the `(from, to)` pair cannot be handled by the built-in
445/// kernels and must dispatch through a color management plugin.
446///
447/// Today this fires when either side's [`color_model`](PixelDescriptor::color_model)
448/// is outside the native set (currently just CMYK; future variants —
449/// Lab / XYZ / spot inks — will plug in here). The companion
450/// [`ConvertError::NeedsCms`] is what entry points return when this is
451/// true and no `cms` was passed.
452///
453/// Useful to schedulers: a caller doing batch encode/decode can probe
454/// `requires_cms` once per source/target pair and decide whether to
455/// attach a CMS plugin (e.g. `&MoxCms`) for that batch.
456///
457/// [`color_model`]: zenpixels::PixelDescriptor::color_model
458pub fn requires_cms(from: &PixelDescriptor, to: &PixelDescriptor) -> bool {
459    !native_color_model(from.color_model()) || !native_color_model(to.color_model())
460}
461
462impl ConvertPlan {
463    /// Assemble a plan with the default (no-anchor) PQ scale. The single place
464    /// `pq_anchor_scale` is defaulted, so every construction path starts at the
465    /// behavior-preserving `1.0`.
466    fn build(from: PixelDescriptor, to: PixelDescriptor, steps: Vec<ConvertStep>) -> Self {
467        Self {
468            from,
469            to,
470            steps,
471            pq_anchor_scale: 1.0,
472        }
473    }
474
475    /// Anchor this plan's **PQ** steps to an absolute-luminance white point —
476    /// the cd/m² that relative-linear `1.0` represents (e.g.
477    /// [`DiffuseWhite::BT2408`](zenpixels::hdr::DiffuseWhite::BT2408) = 203).
478    ///
479    /// The PQ kernels then scale by `nits / 10000` across the relative-linear ↔
480    /// PQ-absolute boundary (encode multiplies before the OETF, decode divides
481    /// after the EOTF), so a relative-linear buffer maps to PQ at the right
482    /// brightness without the caller pre-scaling. A decode+encode pair in one
483    /// plan shares the scale and round-trips exactly. The BT.2408 default (203)
484    /// reproduces the byte-parity-verified pre-scale that `quantize_to` used to
485    /// do by hand. HLG steps are unaffected (scene-referred anchoring differs).
486    #[must_use]
487    pub(crate) fn with_pq_anchor(mut self, anchor: zenpixels::hdr::DiffuseWhite) -> Self {
488        // `diffuse_white_nits` / `PQ_PEAK_NITS` makes the unit explicit: the scale
489        // is the fraction of PQ's 10000 cd/m² peak that relative-linear 1.0 sits at.
490        let diffuse_white_nits = f64::from(anchor.nits());
491        const PQ_PEAK_NITS: f64 = 10_000.0;
492        self.pq_anchor_scale = (diffuse_white_nits / PQ_PEAK_NITS) as f32;
493        self
494    }
495
496    /// Create a conversion plan from `from` to `to`.
497    ///
498    /// Returns `Err` if no conversion path exists. A
499    /// [`SignalRange`](zenpixels::SignalRange) mismatch always refuses
500    /// ([`ConvertError::NoPath`]): there are no Narrow↔Full conversion
501    /// kernels, and relabeling without rescaling would corrupt pixels — see
502    /// the signal-range notes on the [crate docs](crate#step-3-convert).
503    ///
504    /// CMYK (and any other non-native color model) returns
505    /// [`ConvertError::NeedsCms`] so the caller can re-issue via
506    /// [`RowConverter::new_explicit_with_cms`](crate::RowConverter::new_explicit_with_cms)
507    /// with a [`PluggableCms`](crate::cms::PluggableCms) backend attached.
508    /// `ConvertPlan` itself never dispatches through CMS — wire the call
509    /// through `RowConverter` for that.
510    #[track_caller]
511    pub fn new(from: PixelDescriptor, to: PixelDescriptor) -> Result<Self, At<ConvertError>> {
512        if requires_cms(&from, &to) {
513            return Err(whereat::at!(ConvertError::NeedsCms { from, to }));
514        }
515        if from == to {
516            return Ok(Self::build(from, to, vec![ConvertStep::Identity]));
517        }
518
519        // Refuse signal-range crossings: no Narrow↔Full steps exist (no
520        // expand/contract kernels), and a plan built from the *other*
521        // descriptor differences would emit the source's range-coded values
522        // under the target's range label — mislabeled pixels (lifted blacks
523        // when narrow data is labeled full, crushed when full data is later
524        // expanded as narrow), not a conversion. Until range kernels land,
525        // range is preserved verbatim or the conversion fails loudly.
526        // Same-range plans (including Narrow→Narrow) are unaffected.
527        if from.signal_range != to.signal_range {
528            return Err(whereat::at!(ConvertError::NoPath { from, to }));
529        }
530
531        // Refuse HLG↔PQ. HLG is scene-referred — these kernels apply only its
532        // OETF, with no OOTF and no `Lw`/peak — while PQ is absolute display
533        // light (cd/m²). Routing one to the other through the shared "linear"
534        // intermediate conflates the two luminance domains by orders of
535        // magnitude (scene-normalized `[0,1]` vs absolute [0,10000 cd/m²]): a
536        // deterministic but grossly **wrong** result. Until the OOTF +
537        // `(diffuse_white, Lw)` threading lands (#45 S2), fail loudly rather than
538        // emit wrong pixels — the same posture as the signal-range refusal.
539        // HLG↔SDR/linear are *not* refused here: those stay within a normalized
540        // domain (endpoint-correct), missing only the mid-tone OOTF gamma.
541        if matches!(
542            (from.transfer(), to.transfer()),
543            (TransferFunction::Hlg, TransferFunction::Pq)
544                | (TransferFunction::Pq, TransferFunction::Hlg)
545        ) {
546            return Err(whereat::at!(ConvertError::NoPath { from, to }));
547        }
548
549        // Refuse HDR → SDR through the plain entry point. The plain plan
550        // builder has no source-peak luminance to thread into the BT.2446-A
551        // curve, and silently routing through `Pq U16 → Linear F32 →
552        // Linear F32 (no tone map) → target` would produce semantically
553        // wrong pixels (any HDR sample above SDR diffuse-white saturates
554        // to 1.0). Force the caller to use the tone-mapped entry point.
555        // HLG↔PQ already refused above; this catches HDR→{Linear, Srgb,
556        // Bt709, Gamma22}. Under `hdr-experimental` only — without it
557        // the variant doesn't exist and the historic pass-through
558        // behavior is preserved as a deliberate semi-compatibility
559        // shim for legacy non-HDR builds.
560        #[cfg(feature = "hdr-experimental")]
561        if is_hdr_to_sdr(from.transfer(), to.transfer()) {
562            return Err(whereat::at!(ConvertError::HdrSourceRequiresPeak {
563                from,
564                to,
565            }));
566        }
567
568        let mut steps = Vec::with_capacity(3);
569
570        // Step 1: Layout conversion (within same depth class).
571        // Step 2: Depth conversion.
572        // Step 3: Alpha mode conversion.
573        //
574        // For cross-depth conversions, we convert layout at the source depth
575        // first, then change depth. This minimizes the number of channels
576        // we need to depth-convert.
577
578        let need_depth_change = from.channel_type() != to.channel_type();
579        let need_layout_change = from.layout() != to.layout();
580        let need_alpha_change =
581            from.alpha() != to.alpha() && from.alpha().is_some() && to.alpha().is_some();
582
583        // Depth/TF steps are needed when depth changes, or when transfer
584        // functions differ (at any depth — integer TF changes route through
585        // an F32 linear intermediate, handled in `depth_steps`).
586        let need_depth_or_tf = need_depth_change || from.transfer() != to.transfer();
587
588        // If we need to change depth AND layout, plan the optimal order.
589        if need_layout_change {
590            // When going to fewer channels, convert layout first (less depth work).
591            // When going to more channels, convert depth first (less layout work).
592            //
593            // Exception: Oklab layout steps require f32 data. When the source
594            // is integer (U8/U16) and the layout change involves Oklab, we must
595            // convert depth first regardless of channel count.
596            let src_ch = from.layout().channels();
597            let dst_ch = to.layout().channels();
598            let involves_oklab =
599                matches!(from.layout(), ChannelLayout::Oklab | ChannelLayout::OklabA)
600                    || matches!(to.layout(), ChannelLayout::Oklab | ChannelLayout::OklabA);
601
602            // Oklab conversion requires known primaries for the RGB→LMS matrix.
603            if involves_oklab && from.primaries == ColorPrimaries::Unknown {
604                return Err(whereat::at!(ConvertError::NoPath { from, to }));
605            }
606
607            let depth_first = need_depth_or_tf
608                && (dst_ch > src_ch || (involves_oklab && from.channel_type() != ChannelType::F32));
609
610            if depth_first {
611                // Depth first, then layout.
612                steps.extend(
613                    depth_steps(
614                        from.channel_type(),
615                        to.channel_type(),
616                        from.transfer(),
617                        to.transfer(),
618                    )
619                    .map_err(|e| whereat::at!(e))?,
620                );
621                steps.extend(layout_steps(from.layout(), to.layout()));
622            } else {
623                // Layout first, then depth.
624                steps.extend(layout_steps(from.layout(), to.layout()));
625                if need_depth_or_tf {
626                    steps.extend(
627                        depth_steps(
628                            from.channel_type(),
629                            to.channel_type(),
630                            from.transfer(),
631                            to.transfer(),
632                        )
633                        .map_err(|e| whereat::at!(e))?,
634                    );
635                }
636            }
637        } else if need_depth_or_tf {
638            steps.extend(
639                depth_steps(
640                    from.channel_type(),
641                    to.channel_type(),
642                    from.transfer(),
643                    to.transfer(),
644                )
645                .map_err(|e| whereat::at!(e))?,
646            );
647        }
648
649        // Alpha mode conversion (if both have alpha and modes differ).
650        if need_alpha_change {
651            match (from.alpha(), to.alpha()) {
652                (Some(AlphaMode::Straight), Some(AlphaMode::Premultiplied)) => {
653                    steps.push(ConvertStep::StraightToPremul);
654                }
655                (Some(AlphaMode::Premultiplied), Some(AlphaMode::Straight)) => {
656                    steps.push(ConvertStep::PremulToStraight);
657                }
658                _ => {}
659            }
660        }
661
662        // Primaries conversion: if source and destination have different known
663        // primaries, inject a gamut matrix in linear f32 space.
664        let need_primaries = from.primaries != to.primaries
665            && from.primaries != ColorPrimaries::Unknown
666            && to.primaries != ColorPrimaries::Unknown;
667
668        if need_primaries
669            && let Some(matrix) = crate::gamut::conversion_matrix(from.primaries, to.primaries)
670        {
671            // Flatten the 3×3 matrix for storage in the step enum.
672            let flat = [
673                matrix[0][0],
674                matrix[0][1],
675                matrix[0][2],
676                matrix[1][0],
677                matrix[1][1],
678                matrix[1][2],
679                matrix[2][0],
680                matrix[2][1],
681                matrix[2][2],
682            ];
683
684            // The gamut matrix must be applied in linear f32 space.
685            // Check if the existing steps already go through linear f32.
686            let mut goes_through_linear = false;
687            {
688                let mut desc = from;
689                for step in &steps {
690                    desc = intermediate_desc(desc, step);
691                    if desc.channel_type() == ChannelType::F32
692                        && desc.transfer() == TransferFunction::Linear
693                    {
694                        goes_through_linear = true;
695                    }
696                }
697            }
698
699            if goes_through_linear {
700                // Insert the gamut matrix right after the first step that
701                // produces linear f32. All subsequent steps encode to the
702                // target format.
703                let mut insert_pos = 0;
704                let mut desc = from;
705                for (i, step) in steps.iter().enumerate() {
706                    desc = intermediate_desc(desc, step);
707                    if desc.channel_type() == ChannelType::F32
708                        && desc.transfer() == TransferFunction::Linear
709                    {
710                        insert_pos = i + 1;
711                        break;
712                    }
713                }
714                let gamut_step = if desc.layout().has_alpha() {
715                    ConvertStep::GamutMatrixRgbaF32(flat)
716                } else {
717                    ConvertStep::GamutMatrixRgbF32(flat)
718                };
719                steps.insert(insert_pos, gamut_step);
720            } else {
721                // No existing linear f32 step — we must add linearize → gamut → delinearize.
722                // Determine layout for the gamut step.
723                let has_alpha = from.layout().has_alpha() || to.layout().has_alpha();
724                // Use the layout at the current point in the plan.
725                let mut desc = from;
726                for step in &steps {
727                    desc = intermediate_desc(desc, step);
728                }
729                let gamut_step = if desc.layout().has_alpha() || has_alpha {
730                    ConvertStep::GamutMatrixRgbaF32(flat)
731                } else {
732                    ConvertStep::GamutMatrixRgbF32(flat)
733                };
734
735                // Insert linearize → gamut → encode-to-target-tf at the end,
736                // before any alpha mode steps.
737                let linearize = match desc.transfer() {
738                    TransferFunction::Srgb => ConvertStep::SrgbF32ToLinearF32,
739                    TransferFunction::Bt709 => ConvertStep::Bt709F32ToLinearF32,
740                    TransferFunction::Pq => ConvertStep::PqF32ToLinearF32,
741                    TransferFunction::Hlg => ConvertStep::HlgF32ToLinearF32,
742                    TransferFunction::Gamma22 => ConvertStep::Gamma22F32ToLinearF32,
743                    TransferFunction::Linear => ConvertStep::Identity,
744                    _ => ConvertStep::SrgbF32ToLinearF32, // assume sRGB for Unknown
745                };
746                let to_target_tf = match to.transfer() {
747                    TransferFunction::Srgb => ConvertStep::LinearF32ToSrgbF32,
748                    TransferFunction::Bt709 => ConvertStep::LinearF32ToBt709F32,
749                    TransferFunction::Pq => ConvertStep::LinearF32ToPqF32,
750                    TransferFunction::Hlg => ConvertStep::LinearF32ToHlgF32,
751                    TransferFunction::Gamma22 => ConvertStep::LinearF32ToGamma22F32,
752                    TransferFunction::Linear => ConvertStep::Identity,
753                    _ => ConvertStep::LinearF32ToSrgbF32, // assume sRGB for Unknown
754                };
755
756                // Need to be in f32 first. If current is integer, add naive conversion.
757                let mut gamut_steps = Vec::new();
758                // Direct fused-step emissions for common cases.
759                if desc.channel_type() == ChannelType::U16
760                    && desc.transfer() == TransferFunction::Srgb
761                    && to.channel_type() == ChannelType::U16
762                    && to.transfer() == TransferFunction::Srgb
763                    && !desc.layout().has_alpha()
764                    && !to.layout().has_alpha()
765                {
766                    // u16 sRGB → u16 sRGB RGB: single-step matlut.
767                    gamut_steps.push(ConvertStep::Fused {
768                        kind: FusedKind::SrgbU16GamutRgb,
769                        matrix: flat,
770                    });
771                    steps.extend(gamut_steps);
772                    if steps.is_empty() {
773                        steps.push(ConvertStep::Identity);
774                    }
775                    fuse_matlut_patterns(&mut steps);
776                    return Ok(Self::build(from, to, steps));
777                }
778                if desc.channel_type() == ChannelType::U8
779                    && matches!(desc.transfer(), TransferFunction::Srgb)
780                    && to.channel_type() == ChannelType::F32
781                    && to.transfer() == TransferFunction::Linear
782                    && !desc.layout().has_alpha()
783                    && !to.layout().has_alpha()
784                {
785                    // u8 sRGB → linear f32 RGB: cross-depth matlut.
786                    gamut_steps.push(ConvertStep::Fused {
787                        kind: FusedKind::SrgbU8ToLinearF32Rgb,
788                        matrix: flat,
789                    });
790                    steps.extend(gamut_steps);
791                    if steps.is_empty() {
792                        steps.push(ConvertStep::Identity);
793                    }
794                    fuse_matlut_patterns(&mut steps);
795                    return Ok(Self::build(from, to, steps));
796                }
797                if desc.channel_type() == ChannelType::F32
798                    && desc.transfer() == TransferFunction::Linear
799                    && to.channel_type() == ChannelType::U8
800                    && to.transfer() == TransferFunction::Srgb
801                    && !desc.layout().has_alpha()
802                    && !to.layout().has_alpha()
803                {
804                    // linear f32 → u8 sRGB RGB: cross-depth matlut.
805                    gamut_steps.push(ConvertStep::Fused {
806                        kind: FusedKind::LinearF32ToSrgbU8Rgb,
807                        matrix: flat,
808                    });
809                    steps.extend(gamut_steps);
810                    if steps.is_empty() {
811                        steps.push(ConvertStep::Identity);
812                    }
813                    fuse_matlut_patterns(&mut steps);
814                    return Ok(Self::build(from, to, steps));
815                }
816                if desc.channel_type() != ChannelType::F32 {
817                    // Use the fused sRGB u8→linear f32 if applicable.
818                    if desc.channel_type() == ChannelType::U8
819                        && matches!(
820                            desc.transfer(),
821                            TransferFunction::Srgb
822                                | TransferFunction::Bt709
823                                | TransferFunction::Unknown
824                        )
825                    {
826                        gamut_steps.push(ConvertStep::SrgbU8ToLinearF32);
827                        // Already linear, skip separate linearize.
828                        gamut_steps.push(gamut_step);
829                        gamut_steps.push(ConvertStep::LinearF32ToSrgbU8);
830                    } else if desc.channel_type() == ChannelType::U16
831                        && desc.transfer() == TransferFunction::Pq
832                    {
833                        gamut_steps.push(ConvertStep::PqU16ToLinearF32);
834                        gamut_steps.push(gamut_step);
835                        gamut_steps.push(ConvertStep::LinearF32ToPqU16);
836                    } else if desc.channel_type() == ChannelType::U16
837                        && desc.transfer() == TransferFunction::Hlg
838                    {
839                        gamut_steps.push(ConvertStep::HlgU16ToLinearF32);
840                        gamut_steps.push(gamut_step);
841                        gamut_steps.push(ConvertStep::LinearF32ToHlgU16);
842                    } else {
843                        // Generic: naive to f32, linearize, gamut, delinearize, naive back
844                        gamut_steps.push(ConvertStep::NaiveU8ToF32);
845                        if !matches!(linearize, ConvertStep::Identity) {
846                            gamut_steps.push(linearize);
847                        }
848                        gamut_steps.push(gamut_step);
849                        if !matches!(to_target_tf, ConvertStep::Identity) {
850                            gamut_steps.push(to_target_tf);
851                        }
852                        gamut_steps.push(ConvertStep::NaiveF32ToU8);
853                    }
854                } else {
855                    // Already f32, just linearize → gamut → encode
856                    if !matches!(linearize, ConvertStep::Identity) {
857                        gamut_steps.push(linearize);
858                    }
859                    gamut_steps.push(gamut_step);
860                    if !matches!(to_target_tf, ConvertStep::Identity) {
861                        gamut_steps.push(to_target_tf);
862                    }
863                }
864
865                steps.extend(gamut_steps);
866            }
867        }
868
869        if steps.is_empty() {
870            // Transfer-only difference or alpha-mode-only: identity path.
871            steps.push(ConvertStep::Identity);
872        }
873
874        // Peephole fusion: collapse common 3-step patterns into single fused
875        // kernels that avoid scratch-buffer round-trips.
876        fuse_matlut_patterns(&mut steps);
877
878        Ok(Self::build(from, to, steps))
879    }
880
881    /// Create an HDR→SDR conversion plan with the given source-peak
882    /// luminance.
883    ///
884    /// Equivalent to [`ConvertPlan::new_with_hdr_config`] called with
885    /// [`HdrConfig::for_source_peak(source_peak_nits)`](HdrConfig::for_source_peak)
886    /// (`target_peak_nits = 100.0`, `gamut_knee = 0.96`).
887    ///
888    /// The plan inserts a [`Bt2446A`](crate::hdr::Bt2446A) tone-map step
889    /// (and an OKLch soft-compress step for non-BT.2020 targets) into the
890    /// usual transfer / depth / gamut chain. Non-HDR conversions go through
891    /// the same path as [`ConvertPlan::new`].
892    ///
893    /// # Errors
894    ///
895    /// Same as [`ConvertPlan::new`] for non-HDR conversions.
896    /// [`HdrSourceRequiresPeak`] is raised only when `source_peak_nits`
897    /// is not a positive, finite number — a supplied-but-degenerate peak
898    /// would otherwise tone-map every pixel to black.
899    ///
900    /// [`HdrSourceRequiresPeak`]: ConvertError::HdrSourceRequiresPeak
901    ///
902    /// # Panics
903    ///
904    /// Same panics as [`ConvertPlan::new`] (CMYK descriptors).
905    #[cfg(feature = "hdr-experimental")]
906    #[track_caller]
907    pub fn new_with_hdr_peak(
908        from: PixelDescriptor,
909        to: PixelDescriptor,
910        source_peak_nits: f32,
911    ) -> Result<Self, At<ConvertError>> {
912        Self::new_with_hdr_config(from, to, HdrConfig::for_source_peak(source_peak_nits))
913    }
914
915    /// Create an HDR→SDR conversion plan with full knob control.
916    ///
917    /// On HDR→SDR conversions (`Pq` / `Hlg` source → SDR target, OR a
918    /// `Linear` source where the caller declares HDR semantics via this
919    /// constructor) inserts:
920    ///
921    /// 1. HDR transfer decode (PQ/HLG → linear) — same kernels as
922    ///    [`ConvertPlan::new`]. Skipped when the source is already
923    ///    `Linear`.
924    /// 2. Source primaries → BT.2020 matrix (skipped when source is BT.2020).
925    /// 3. `ToneMapBt2446A` step (the BT.2446 Method A curve operating in
926    ///    BT.2020 RGB).
927    /// 4. BT.2020 → target primaries matrix (skipped when target is BT.2020).
928    /// 5. `SoftCompressOklch` step (skipped when target is BT.2020 —
929    ///    wide-gamut output mode preserves chroma).
930    /// 6. Linear → target transfer encode + any depth conversion (sRGB u8,
931    ///    BT.1886 f32, etc.) — same kernels as [`ConvertPlan::new`].
932    ///
933    /// For sources that are obviously SDR (`Srgb` / `Bt709` / `Gamma22`)
934    /// the `hdr` argument is ignored and this returns the same plan
935    /// [`ConvertPlan::new`] would build — no tone-map gets injected into
936    /// a path that doesn't need one.
937    ///
938    // ToneMapBt2446A / SoftCompressOklch are crate-internal `ConvertStep` variants
939    // — referenced by name in the prose above; explicit links would point at
940    // private items.
941    ///
942    /// # Errors
943    ///
944    /// Same as [`ConvertPlan::new`] for non-HDR conversions. For HDR
945    /// sources, [`HdrSourceRequiresPeak`] is raised when
946    /// `hdr.source_peak_nits` or `hdr.target_peak_nits` is not a
947    /// positive, finite number (including the unset
948    /// [`HdrConfig::default`] value `0.0`) — degenerate peaks would
949    /// otherwise flow into the BT.2446-A constants as `inf`/NaN and the
950    /// kernel's NaN scrub would silently emit an all-black image.
951    ///
952    /// [`HdrSourceRequiresPeak`]: ConvertError::HdrSourceRequiresPeak
953    ///
954    /// CMYK (and any other non-native color model) returns
955    /// [`ConvertError::NeedsCms`] — same posture as
956    /// [`ConvertPlan::new`]. HDR tone-mapping is RGB-only; a CMS is the
957    /// right tool for CMYK↔RGB even on the HDR construction path.
958    #[cfg(feature = "hdr-experimental")]
959    #[track_caller]
960    pub fn new_with_hdr_config(
961        from: PixelDescriptor,
962        to: PixelDescriptor,
963        hdr: HdrConfig,
964    ) -> Result<Self, At<ConvertError>> {
965        if requires_cms(&from, &to) {
966            return Err(whereat::at!(ConvertError::NeedsCms { from, to }));
967        }
968        // SDR source paths take the regular plan path — calling the
969        // HDR-aware constructor on (e.g.) sRGB → sRGB shouldn't force a
970        // tone map. The HDR pipeline runs for PQ/HLG sources AND for
971        // `Linear` sources (the caller may have a Linear-tagged
972        // gain-map-reconstructed HDR buffer; the constructor's name is
973        // the opt-in signal).
974        let src_is_sdr_encoded = matches!(
975            from.transfer(),
976            TransferFunction::Srgb | TransferFunction::Bt709 | TransferFunction::Gamma22
977        );
978        if src_is_sdr_encoded {
979            return Self::new(from, to);
980        }
981        // Reject unusable peak luminances up front (only on the HDR path —
982        // the SDR early-return above documents `hdr` as ignored there).
983        // `HdrConfig::default()` ships `source_peak_nits = 0.0` ("unset");
984        // before this guard a zero / negative / non-finite peak flowed
985        // into the BT.2446-A constants (`1 / ln(1) = inf`, `powf` of a
986        // negative → NaN) and the tone-map kernel's NaN scrub then emitted
987        // a fully BLACK image with no error — silent total pixel loss.
988        let peak_usable = |v: f32| v.is_finite() && v > 0.0;
989        if !peak_usable(hdr.source_peak_nits) || !peak_usable(hdr.target_peak_nits) {
990            return Err(whereat::at!(ConvertError::HdrSourceRequiresPeak {
991                from,
992                to
993            }));
994        }
995        // Note: do NOT early-return on `from == to`. The HDR-aware
996        // constructor is the caller's opt-in signal that the source carries
997        // HDR semantics — even when the source and target descriptors are
998        // byte-identical (e.g. both `RGBF32_LINEAR`), the tone-map +
999        // gamut-compress chain still needs to run. Identity bytes-out
1000        // would silently skip the HDR work the constructor was called to
1001        // perform.
1002
1003        // Same signal-range posture as `new` — Narrow↔Full crossings refuse
1004        // because no kernels exist yet.
1005        if from.signal_range != to.signal_range {
1006            return Err(whereat::at!(ConvertError::NoPath { from, to }));
1007        }
1008
1009        // The pipeline: src → linear-F32-in-source-primaries → (source→BT.2020)
1010        // → ToneMap → (BT.2020→target) → SoftCompress → target-encode.
1011        // The intermediate descriptor between steps is linear-light F32 in
1012        // some primaries, with the source's layout (RGB or RGBA) carried
1013        // through (alpha passthrough at every step). We let the existing
1014        // depth_steps build the decode side, then append our HDR steps,
1015        // then let the existing encode chain finish.
1016        let mut steps: Vec<ConvertStep> = Vec::with_capacity(8);
1017
1018        // ---- (a) Decode source transfer → F32 linear. Reuse `depth_steps`
1019        // with intermediate target = (F32, source.layout(), source.alpha(),
1020        // Linear) — this emits the right PQ/HLG/cross-depth kernels.
1021        let after_decode = PixelDescriptor::new(
1022            ChannelType::F32,
1023            from.layout(),
1024            from.alpha(),
1025            TransferFunction::Linear,
1026        );
1027        steps.extend(
1028            depth_steps(
1029                from.channel_type(),
1030                ChannelType::F32,
1031                from.transfer(),
1032                TransferFunction::Linear,
1033            )
1034            .map_err(|e| whereat::at!(e))?,
1035        );
1036
1037        // ---- (b) Source primaries → BT.2020 (skip when source IS BT.2020).
1038        if from.primaries != ColorPrimaries::Bt2020
1039            && let Some(matrix) =
1040                crate::gamut::conversion_matrix(from.primaries, ColorPrimaries::Bt2020)
1041        {
1042            let flat = [
1043                matrix[0][0],
1044                matrix[0][1],
1045                matrix[0][2],
1046                matrix[1][0],
1047                matrix[1][1],
1048                matrix[1][2],
1049                matrix[2][0],
1050                matrix[2][1],
1051                matrix[2][2],
1052            ];
1053            let step = if after_decode.layout().has_alpha() {
1054                ConvertStep::GamutMatrixRgbaF32(flat)
1055            } else {
1056                ConvertStep::GamutMatrixRgbF32(flat)
1057            };
1058            steps.push(step);
1059        }
1060
1061        // ---- (c) BT.2446 Method A tone map (BT.2020 HDR → BT.2020 SDR).
1062        steps.push(ConvertStep::ToneMapBt2446A {
1063            source_peak_nits: hdr.source_peak_nits,
1064            target_peak_nits: hdr.target_peak_nits,
1065        });
1066
1067        // ---- (d) BT.2020 → target primaries (skip when target IS BT.2020).
1068        if to.primaries != ColorPrimaries::Bt2020
1069            && to.primaries != ColorPrimaries::Unknown
1070            && let Some(matrix) =
1071                crate::gamut::conversion_matrix(ColorPrimaries::Bt2020, to.primaries)
1072        {
1073            let flat = [
1074                matrix[0][0],
1075                matrix[0][1],
1076                matrix[0][2],
1077                matrix[1][0],
1078                matrix[1][1],
1079                matrix[1][2],
1080                matrix[2][0],
1081                matrix[2][1],
1082                matrix[2][2],
1083            ];
1084            let step = if after_decode.layout().has_alpha() {
1085                ConvertStep::GamutMatrixRgbaF32(flat)
1086            } else {
1087                ConvertStep::GamutMatrixRgbF32(flat)
1088            };
1089            steps.push(step);
1090        }
1091
1092        // ---- (e) OKLch soft chroma compression (skip when target IS BT.2020;
1093        // wide-gamut output mode preserves chroma).
1094        if to.primaries != ColorPrimaries::Bt2020 && to.primaries != ColorPrimaries::Unknown {
1095            steps.push(ConvertStep::SoftCompressOklch {
1096                primaries: to.primaries,
1097                knee: hdr.gamut_knee,
1098            });
1099        }
1100
1101        // ---- (f) Layout conversion (e.g., RGBA→RGB DropAlpha), if any.
1102        // After the HDR steps we're still in (F32, from.layout(), from.alpha(),
1103        // Linear) carrying target.primaries (last gamut matrix updated them).
1104        if from.layout() != to.layout() {
1105            steps.extend(layout_steps(from.layout(), to.layout()));
1106        }
1107
1108        // ---- (g) Linear F32 → target transfer + depth. Re-use depth_steps
1109        // for the F32 → target.channel_type leg with the encode TF.
1110        let need_depth_or_tf_encode =
1111            to.channel_type() != ChannelType::F32 || to.transfer() != TransferFunction::Linear;
1112        if need_depth_or_tf_encode {
1113            steps.extend(
1114                depth_steps(
1115                    ChannelType::F32,
1116                    to.channel_type(),
1117                    TransferFunction::Linear,
1118                    to.transfer(),
1119                )
1120                .map_err(|e| whereat::at!(e))?,
1121            );
1122        }
1123
1124        // ---- (h) Alpha mode (Straight↔Premultiplied).
1125        //
1126        // KNOWN LIMITATION (premultiplied HDR sources): steps (b)–(e) above —
1127        // including the NONLINEAR tone-map and OKLch soft-compress — run on the
1128        // source's alpha mode carried through from step (a). Linear ops (the
1129        // gamut matrices) commute with premultiplication, but the nonlinear
1130        // tone-map does not: `TM(α·R) ≠ α·TM(R)`. A `Premultiplied` source is
1131        // therefore tone-mapped on premultiplied values, and this step only
1132        // reconciles the alpha *mode* afterwards. Correct handling needs an
1133        // unpremultiply before step (b) and a re-premultiply here — but the
1134        // library's premul convention is encoded-space (Canvas 2D; see the
1135        // `new_explicit` MatteComposite note), so doing it in the linear
1136        // pipeline is subtle and deferred rather than done wrong. In practice
1137        // PQ/HLG sources are virtually always straight/opaque; premultiplied
1138        // HDR is the rare case. Tracked for the `hdr-experimental` stabilization.
1139        if from.alpha() != to.alpha() && from.alpha().is_some() && to.alpha().is_some() {
1140            match (from.alpha(), to.alpha()) {
1141                (Some(AlphaMode::Straight), Some(AlphaMode::Premultiplied)) => {
1142                    steps.push(ConvertStep::StraightToPremul);
1143                }
1144                (Some(AlphaMode::Premultiplied), Some(AlphaMode::Straight)) => {
1145                    steps.push(ConvertStep::PremulToStraight);
1146                }
1147                _ => {}
1148            }
1149        }
1150
1151        if steps.is_empty() {
1152            steps.push(ConvertStep::Identity);
1153        }
1154
1155        Ok(Self::build(from, to, steps))
1156    }
1157
1158    /// Create a conversion plan with explicit policy enforcement.
1159    ///
1160    /// Validates that the planned conversion steps are allowed by the given
1161    /// policies before creating the plan. Returns an error if a forbidden
1162    /// operation would be required.
1163    ///
1164    /// CMYK (and any other non-native color model) returns
1165    /// [`ConvertError::NeedsCms`] — same posture as
1166    /// [`ConvertPlan::new`]. To dispatch CMYK ↔ RGB through a CMS, build
1167    /// the converter via
1168    /// [`RowConverter::new_explicit_with_cms`](crate::RowConverter::new_explicit_with_cms)
1169    /// with a [`PluggableCms`](crate::cms::PluggableCms) plugin attached.
1170    #[track_caller]
1171    pub fn new_explicit(
1172        from: PixelDescriptor,
1173        to: PixelDescriptor,
1174        options: &ConvertOptions,
1175    ) -> Result<Self, At<ConvertError>> {
1176        if requires_cms(&from, &to) {
1177            return Err(whereat::at!(ConvertError::NeedsCms { from, to }));
1178        }
1179        // Check alpha removal policy.
1180        let drops_alpha = from.alpha().is_some() && to.alpha().is_none();
1181        if drops_alpha && options.alpha_policy == AlphaPolicy::Forbid {
1182            return Err(whereat::at!(ConvertError::AlphaRemovalForbidden));
1183        }
1184
1185        // Check depth reduction policy. Compare by precision bits, not byte
1186        // size — F16 and U16 are both 2 bytes but F16 carries only ~11 bits of
1187        // precision vs U16's 16, so a U16→F16 hop IS a precision reduction and
1188        // must be policy-gated.
1189        let reduces_depth = crate::negotiate::channel_bits(from.channel_type())
1190            > crate::negotiate::channel_bits(to.channel_type());
1191        if reduces_depth && options.depth_policy == DepthPolicy::Forbid {
1192            return Err(whereat::at!(ConvertError::DepthReductionForbidden));
1193        }
1194
1195        // Check RGB→Gray requires luma coefficients.
1196        let src_is_rgb = matches!(
1197            from.layout(),
1198            ChannelLayout::Rgb | ChannelLayout::Rgba | ChannelLayout::Bgra
1199        );
1200        let dst_is_gray = matches!(to.layout(), ChannelLayout::Gray | ChannelLayout::GrayAlpha);
1201        if src_is_rgb && dst_is_gray && options.luma.is_none() {
1202            return Err(whereat::at!(ConvertError::RgbToGray));
1203        }
1204
1205        let mut plan = Self::new(from, to).at()?;
1206
1207        // Replace DropAlpha with MatteComposite when policy is CompositeOnto.
1208        //
1209        // The `matte_composite` kernel uses the straight-alpha over operator
1210        // `fg*a + bg*(1-a)`, linearizing the sRGB matte and pixel RGB
1211        // per-pixel using the source TF (kernel-side TF dispatch via the
1212        // `MatteTf` trait). Alpha stays as-is (alpha is always linear,
1213        // regardless of color-channel TF).
1214        //
1215        // One planner-side caveat handled here:
1216        //
1217        // **Premultiplied source.** If the source is premultiplied (our
1218        // library's convention is encoded-space premul, per Canvas 2D),
1219        // the straight kernel would multiply by `a` twice:
1220        // `straight*a² + bg*(1-a)`. Fix: insert `PremulToStraight` before
1221        // `MatteComposite`.
1222        //
1223        // We deliberately do NOT wrap with `SrgbF32ToLinearF32` /
1224        // `LinearF32ToSrgbF32` to handle non-linear data: those steps
1225        // linearize alpha too, which breaks the blend math.
1226        if drops_alpha && let AlphaPolicy::CompositeOnto { r, g, b } = options.alpha_policy {
1227            let src_is_premul = from.alpha() == Some(AlphaMode::Premultiplied);
1228            let mut idx = 0;
1229            while idx < plan.steps.len() {
1230                if matches!(plan.steps[idx], ConvertStep::DropAlpha) {
1231                    plan.steps[idx] = ConvertStep::MatteComposite { r, g, b };
1232                    if src_is_premul {
1233                        plan.steps.insert(idx, ConvertStep::PremulToStraight);
1234                        idx += 1;
1235                    }
1236                }
1237                idx += 1;
1238            }
1239        }
1240
1241        // When the caller opts out of clipping, swap pure-f32 sRGB transfer
1242        // steps for their sign-preserving extended-range counterparts.
1243        // Fused u8/u16 matlut steps are unaffected (integer I/O can't
1244        // represent extended range anyway).
1245        if !options.clip_out_of_gamut {
1246            for step in &mut plan.steps {
1247                match step {
1248                    ConvertStep::SrgbF32ToLinearF32 => {
1249                        *step = ConvertStep::SrgbF32ToLinearF32Extended;
1250                    }
1251                    ConvertStep::LinearF32ToSrgbF32 => {
1252                        *step = ConvertStep::LinearF32ToSrgbF32Extended;
1253                    }
1254                    _ => {}
1255                }
1256            }
1257        }
1258
1259        // Resolve luma coefficients on RgbToGray / RgbaToGray steps. The
1260        // None case was rejected above (line 636), so unwrap is safe here.
1261        // `layout_steps` constructs these variants with a Bt709 placeholder
1262        // because it has no access to options; we replace with the user's
1263        // explicit choice (or the permissive default of Bt709) here.
1264        let user_luma = options.luma.unwrap_or(LumaCoefficients::Bt709);
1265        for step in &mut plan.steps {
1266            match step {
1267                ConvertStep::RgbToGray { coefficients }
1268                | ConvertStep::RgbaToGray { coefficients } => {
1269                    *coefficients = user_luma;
1270                }
1271                _ => {}
1272            }
1273        }
1274
1275        Ok(plan)
1276    }
1277
1278    /// Create a shell plan that records from/to but has no conversion steps.
1279    ///
1280    /// Used when an external CMS transform handles the conversion — the
1281    /// plan exists only for `from()`/`to()` metadata; the actual row
1282    /// work is driven by the external transform stored on `RowConverter`.
1283    pub(crate) fn identity(from: PixelDescriptor, to: PixelDescriptor) -> Self {
1284        Self::build(from, to, vec![ConvertStep::Identity])
1285    }
1286
1287    /// Compose two plans into one: apply `self` then `other`.
1288    ///
1289    /// The composed plan executes both conversions in a single `convert_row`
1290    /// call, using one intermediate buffer instead of two. Adjacent inverse
1291    /// steps are cancelled (e.g., `SrgbU8ToLinearF32` + `LinearF32ToSrgbU8`
1292    /// → identity).
1293    ///
1294    /// Returns `None` if `self.to` != `other.from` (incompatible plans).
1295    pub fn compose(&self, other: &Self) -> Option<Self> {
1296        if self.to != other.from {
1297            return None;
1298        }
1299
1300        let mut steps = self.steps.clone();
1301
1302        // Append other's steps, skipping its Identity if present.
1303        for step in &other.steps {
1304            if matches!(step, ConvertStep::Identity) {
1305                continue;
1306            }
1307            steps.push(step.clone());
1308        }
1309
1310        // Peephole: cancel adjacent inverse pairs.
1311        let mut changed = true;
1312        while changed {
1313            changed = false;
1314            let mut i = 0;
1315            while i + 1 < steps.len() {
1316                if are_inverse(&steps[i], &steps[i + 1]) {
1317                    steps.remove(i + 1);
1318                    steps.remove(i);
1319                    changed = true;
1320                    // Don't advance — check the new adjacent pair.
1321                } else {
1322                    i += 1;
1323                }
1324            }
1325        }
1326
1327        // If everything cancelled, produce identity.
1328        if steps.is_empty() {
1329            steps.push(ConvertStep::Identity);
1330        }
1331
1332        // Remove leading/trailing Identity if there are real steps.
1333        if steps.len() > 1 {
1334            steps.retain(|s| !matches!(s, ConvertStep::Identity));
1335            if steps.is_empty() {
1336                steps.push(ConvertStep::Identity);
1337            }
1338        }
1339
1340        // Composition runs at plan-build time, before any anchor is attached
1341        // (`with_pq_anchor` is applied to the finished plan), so both inputs
1342        // carry the default scale; the merged plan does too.
1343        Some(Self::build(self.from, other.to, steps))
1344    }
1345
1346    /// True if conversion is a no-op.
1347    #[must_use]
1348    pub fn is_identity(&self) -> bool {
1349        self.steps.len() == 1 && matches!(self.steps[0], ConvertStep::Identity)
1350    }
1351
1352    /// Maximum bytes-per-pixel across all intermediate formats in the plan.
1353    ///
1354    /// Used to pre-allocate scratch buffers for streaming conversion.
1355    pub(crate) fn max_intermediate_bpp(&self) -> usize {
1356        let mut desc = self.from;
1357        let mut max_bpp = desc.bytes_per_pixel();
1358        for step in &self.steps {
1359            desc = intermediate_desc(desc, step);
1360            max_bpp = max_bpp.max(desc.bytes_per_pixel());
1361        }
1362        max_bpp
1363    }
1364
1365    /// Crate-internal view of the planned step list — exposed for the
1366    /// estimate-API code under `crate::estimate`. NOT public:
1367    /// `ConvertStep` itself is `pub(crate)`.
1368    pub(crate) fn steps(&self) -> &[ConvertStep] {
1369        &self.steps
1370    }
1371
1372    /// Source descriptor.
1373    pub fn from(&self) -> PixelDescriptor {
1374        self.from
1375    }
1376
1377    /// Target descriptor.
1378    pub fn to(&self) -> PixelDescriptor {
1379        self.to
1380    }
1381
1382    /// Estimate resources for executing this plan on `image` under the
1383    /// given [`ComputeEnvironment`](crate::estimate::ComputeEnvironment).
1384    /// Returns a [`ResourceEstimate`](crate::estimate::ResourceEstimate)
1385    /// whose type shape matches `zencodec::estimate::ResourceEstimate` so
1386    /// codec-side encode/decode estimates can be wired through a multi-
1387    /// stage pipeline at the codec boundary with a trivial conversion.
1388    ///
1389    /// Calibrated from `benches/t1_layout`, `t2_depth`, `t3_tf_fused`,
1390    /// `t4_tf_f32`, `t5_alpha`, `t6_oklab`, `t7_gamut` steady-state
1391    /// throughput; best-effort, ±30 % on the reference machine
1392    /// (Ryzen 9 7950X, AVX2). Real wall time varies with contention,
1393    /// frequency scaling, and CPU model. Identity at 0×0 returns a
1394    /// zero-cost estimate.
1395    ///
1396    /// `peak_memory_bytes_est` is the destination buffer plus row-sized
1397    /// ping-pong scratch (multi-step plans). It does NOT include the
1398    /// caller's persistent state. `intermediate_buffer_count` reports the
1399    /// number of full-image intermediate buffers held simultaneously
1400    /// (0 for identity / single-step plans; 2 for multi-step plans using
1401    /// ping-pong scratch) so schedulers can distinguish 1-giant-buffer plans
1402    /// from N-medium-buffer plans for paging-pressure decisions.
1403    /// `wall_ms` is divided down by `compute.cores()` via the plan's
1404    /// internal threading-bottleneck model (see the [`estimate`] module
1405    /// docs): any SERIAL step forces the whole plan SERIAL; otherwise the
1406    /// smallest per-step knee — `rows / 64` clamped to `[1, 16]` — caps
1407    /// the useful thread count.
1408    ///
1409    /// [`estimate`]: crate::estimate
1410    ///
1411    /// `compute.simd_tier()` applies a coarse per-tier wall-time
1412    /// multiplier on top of the AVX2 baseline (see the `estimate`
1413    /// module docs for the per-tier ratios; TODO per-tier calibration).
1414    ///
1415    /// Cheap to call — walks the plan's steps once and does no
1416    /// allocation. Safe to call repeatedly per-frame in throttled
1417    /// pipelines.
1418    ///
1419    /// For a quick estimate using [`ComputeEnvironment::new()`](crate::estimate::ComputeEnvironment::new)
1420    /// defaults on a `width × height` image, see [`estimate`](Self::estimate).
1421    ///
1422    /// # Example
1423    ///
1424    /// ```rust
1425    /// use zenpixels::PixelDescriptor;
1426    /// use zenpixels_convert::{ComputeEnvironment, ConvertPlan, ImageCharacteristics};
1427    ///
1428    /// let plan = ConvertPlan::new(
1429    ///     PixelDescriptor::RGB8_SRGB,
1430    ///     PixelDescriptor::RGBA8_SRGB,
1431    /// ).unwrap();
1432    /// let image = ImageCharacteristics::new(1920, 1080, PixelDescriptor::RGB8_SRGB);
1433    /// let compute = ComputeEnvironment::new().with_cores(8);
1434    /// let est = plan.estimate_in(&image, &compute);
1435    /// assert!(est.peak_memory_bytes_est().unwrap_or(0) > 0);
1436    /// // wall_ms is `Some(_)` once the plan has measurable work (it can
1437    /// // round to 0 ms for trivial plans, but the field is populated).
1438    /// assert!(est.wall_ms().is_some());
1439    /// ```
1440    #[must_use]
1441    pub fn estimate_in(
1442        &self,
1443        image: &crate::estimate::ImageCharacteristics,
1444        compute: &crate::estimate::ComputeEnvironment,
1445    ) -> crate::estimate::ResourceEstimate {
1446        crate::estimate::estimate_plan(self, image, compute)
1447    }
1448
1449    /// Shortcut: estimate with [`ComputeEnvironment::new()`](crate::estimate::ComputeEnvironment::new)
1450    /// defaults (single core, unknown RAM, unspecified SIMD tier) on a
1451    /// `width × height` image. Builds the
1452    /// [`ImageCharacteristics`](crate::estimate::ImageCharacteristics) from
1453    /// the plan's `from()` descriptor and calls [`estimate_in`](Self::estimate_in).
1454    /// Use [`estimate_in`](Self::estimate_in) directly when the caller has
1455    /// a populated compute environment (e.g.
1456    /// `available_parallelism() + archmage tier`).
1457    ///
1458    /// # Example
1459    ///
1460    /// ```rust
1461    /// use zenpixels::PixelDescriptor;
1462    /// use zenpixels_convert::ConvertPlan;
1463    ///
1464    /// let plan = ConvertPlan::new(
1465    ///     PixelDescriptor::RGB8_SRGB,
1466    ///     PixelDescriptor::RGBA8_SRGB,
1467    /// ).unwrap();
1468    /// let est = plan.estimate(1920, 1080);
1469    /// assert!(est.peak_memory_bytes_est().unwrap_or(0) > 0);
1470    /// assert!(est.wall_ms().is_some());
1471    /// ```
1472    #[must_use]
1473    pub fn estimate(&self, width: u32, height: u32) -> crate::estimate::ResourceEstimate {
1474        let image = crate::estimate::ImageCharacteristics::new(width, height, self.from());
1475        let compute = crate::estimate::ComputeEnvironment::new();
1476        self.estimate_in(&image, &compute)
1477    }
1478}
1479
1480/// Bridge for the [`crate::estimate`] module: mirror of
1481/// [`intermediate_desc`] without making that function public.
1482pub(crate) fn intermediate_desc_for_estimate(
1483    current: PixelDescriptor,
1484    step: &ConvertStep,
1485) -> PixelDescriptor {
1486    intermediate_desc(current, step)
1487}
1488
1489/// Determine the layout conversion step(s).
1490///
1491/// Some layout conversions require two steps (e.g., BGRA -> RGB needs
1492/// swizzle + drop alpha). Returns up to 2 steps.
1493fn layout_steps(from: ChannelLayout, to: ChannelLayout) -> Vec<ConvertStep> {
1494    if from == to {
1495        return Vec::new();
1496    }
1497    match (from, to) {
1498        (ChannelLayout::Bgra, ChannelLayout::Rgba) | (ChannelLayout::Rgba, ChannelLayout::Bgra) => {
1499            vec![ConvertStep::SwizzleBgraRgba]
1500        }
1501        (ChannelLayout::Rgb, ChannelLayout::Rgba) => vec![ConvertStep::AddAlpha],
1502        (ChannelLayout::Rgb, ChannelLayout::Bgra) => {
1503            // Single fused SIMD pass (garb::bytes::rgb_to_bgra). For non-u8
1504            // channel types `apply_step_u8` falls back to AddAlpha+Swizzle.
1505            vec![ConvertStep::RgbToBgra]
1506        }
1507        (ChannelLayout::Rgba, ChannelLayout::Rgb) => vec![ConvertStep::DropAlpha],
1508        (ChannelLayout::Bgra, ChannelLayout::Rgb) => {
1509            // BGRA -> RGBA -> RGB: swizzle then drop alpha.
1510            vec![ConvertStep::SwizzleBgraRgba, ConvertStep::DropAlpha]
1511        }
1512        (ChannelLayout::Gray, ChannelLayout::Rgb) => vec![ConvertStep::GrayToRgb],
1513        (ChannelLayout::Gray, ChannelLayout::Rgba) => vec![ConvertStep::GrayToRgba],
1514        (ChannelLayout::Gray, ChannelLayout::Bgra) => {
1515            // Gray -> RGBA -> BGRA: expand then swizzle.
1516            vec![ConvertStep::GrayToRgba, ConvertStep::SwizzleBgraRgba]
1517        }
1518        (ChannelLayout::Rgb, ChannelLayout::Gray) => vec![ConvertStep::RgbToGray {
1519            coefficients: LumaCoefficients::Bt709,
1520        }],
1521        (ChannelLayout::Rgba, ChannelLayout::Gray) => vec![ConvertStep::RgbaToGray {
1522            coefficients: LumaCoefficients::Bt709,
1523        }],
1524        (ChannelLayout::Bgra, ChannelLayout::Gray) => {
1525            // BGRA -> RGBA -> Gray: swizzle then to gray.
1526            vec![
1527                ConvertStep::SwizzleBgraRgba,
1528                ConvertStep::RgbaToGray {
1529                    coefficients: LumaCoefficients::Bt709,
1530                },
1531            ]
1532        }
1533        (ChannelLayout::GrayAlpha, ChannelLayout::Rgba) => vec![ConvertStep::GrayAlphaToRgba],
1534        (ChannelLayout::GrayAlpha, ChannelLayout::Bgra) => {
1535            // GrayAlpha -> RGBA -> BGRA: expand then swizzle.
1536            vec![ConvertStep::GrayAlphaToRgba, ConvertStep::SwizzleBgraRgba]
1537        }
1538        (ChannelLayout::GrayAlpha, ChannelLayout::Rgb) => vec![ConvertStep::GrayAlphaToRgb],
1539        (ChannelLayout::Gray, ChannelLayout::GrayAlpha) => vec![ConvertStep::GrayToGrayAlpha],
1540        (ChannelLayout::GrayAlpha, ChannelLayout::Gray) => vec![ConvertStep::GrayAlphaToGray],
1541
1542        // Oklab ↔ RGB conversions (via linear RGB).
1543        (ChannelLayout::Rgb, ChannelLayout::Oklab) => vec![ConvertStep::LinearRgbToOklab],
1544        (ChannelLayout::Oklab, ChannelLayout::Rgb) => vec![ConvertStep::OklabToLinearRgb],
1545        (ChannelLayout::Rgba, ChannelLayout::OklabA) => vec![ConvertStep::LinearRgbaToOklaba],
1546        (ChannelLayout::OklabA, ChannelLayout::Rgba) => vec![ConvertStep::OklabaToLinearRgba],
1547
1548        // Oklab ↔ RGB with alpha add/drop.
1549        (ChannelLayout::Rgb, ChannelLayout::OklabA) => {
1550            vec![ConvertStep::AddAlpha, ConvertStep::LinearRgbaToOklaba]
1551        }
1552        (ChannelLayout::OklabA, ChannelLayout::Rgb) => {
1553            vec![ConvertStep::OklabaToLinearRgba, ConvertStep::DropAlpha]
1554        }
1555        (ChannelLayout::Oklab, ChannelLayout::Rgba) => {
1556            vec![ConvertStep::OklabToLinearRgb, ConvertStep::AddAlpha]
1557        }
1558        (ChannelLayout::Rgba, ChannelLayout::Oklab) => {
1559            vec![ConvertStep::DropAlpha, ConvertStep::LinearRgbToOklab]
1560        }
1561
1562        // Oklab ↔ BGRA (swizzle to/from RGBA, then Oklab).
1563        (ChannelLayout::Bgra, ChannelLayout::OklabA) => {
1564            vec![
1565                ConvertStep::SwizzleBgraRgba,
1566                ConvertStep::LinearRgbaToOklaba,
1567            ]
1568        }
1569        (ChannelLayout::OklabA, ChannelLayout::Bgra) => {
1570            vec![
1571                ConvertStep::OklabaToLinearRgba,
1572                ConvertStep::SwizzleBgraRgba,
1573            ]
1574        }
1575        (ChannelLayout::Bgra, ChannelLayout::Oklab) => {
1576            vec![
1577                ConvertStep::SwizzleBgraRgba,
1578                ConvertStep::DropAlpha,
1579                ConvertStep::LinearRgbToOklab,
1580            ]
1581        }
1582        (ChannelLayout::Oklab, ChannelLayout::Bgra) => {
1583            vec![
1584                ConvertStep::OklabToLinearRgb,
1585                ConvertStep::AddAlpha,
1586                ConvertStep::SwizzleBgraRgba,
1587            ]
1588        }
1589
1590        // Gray ↔ Oklab (expand gray to RGB first).
1591        (ChannelLayout::Gray, ChannelLayout::Oklab) => {
1592            vec![ConvertStep::GrayToRgb, ConvertStep::LinearRgbToOklab]
1593        }
1594        (ChannelLayout::Oklab, ChannelLayout::Gray) => {
1595            vec![
1596                ConvertStep::OklabToLinearRgb,
1597                ConvertStep::RgbToGray {
1598                    coefficients: LumaCoefficients::Bt709,
1599                },
1600            ]
1601        }
1602        (ChannelLayout::Gray, ChannelLayout::OklabA) => {
1603            vec![ConvertStep::GrayToRgba, ConvertStep::LinearRgbaToOklaba]
1604        }
1605        (ChannelLayout::OklabA, ChannelLayout::Gray) => {
1606            vec![
1607                ConvertStep::OklabaToLinearRgba,
1608                ConvertStep::RgbaToGray {
1609                    coefficients: LumaCoefficients::Bt709,
1610                },
1611            ]
1612        }
1613        (ChannelLayout::GrayAlpha, ChannelLayout::OklabA) => {
1614            vec![
1615                ConvertStep::GrayAlphaToRgba,
1616                ConvertStep::LinearRgbaToOklaba,
1617            ]
1618        }
1619        (ChannelLayout::OklabA, ChannelLayout::GrayAlpha) => {
1620            // Drop alpha from OklabA→Oklab, convert to RGB, then to GrayAlpha.
1621            // Alpha is lost; this is inherently lossy.
1622            vec![
1623                ConvertStep::OklabaToLinearRgba,
1624                ConvertStep::RgbaToGray {
1625                    coefficients: LumaCoefficients::Bt709,
1626                },
1627                ConvertStep::GrayToGrayAlpha,
1628            ]
1629        }
1630        (ChannelLayout::GrayAlpha, ChannelLayout::Oklab) => {
1631            vec![ConvertStep::GrayAlphaToRgb, ConvertStep::LinearRgbToOklab]
1632        }
1633        (ChannelLayout::Oklab, ChannelLayout::GrayAlpha) => {
1634            vec![
1635                ConvertStep::OklabToLinearRgb,
1636                ConvertStep::RgbToGray {
1637                    coefficients: LumaCoefficients::Bt709,
1638                },
1639                ConvertStep::GrayToGrayAlpha,
1640            ]
1641        }
1642
1643        // Oklab ↔ alpha variants.
1644        (ChannelLayout::Oklab, ChannelLayout::OklabA) => vec![ConvertStep::AddAlpha],
1645        (ChannelLayout::OklabA, ChannelLayout::Oklab) => vec![ConvertStep::DropAlpha],
1646
1647        _ => Vec::new(), // Unsupported layout conversion.
1648    }
1649}
1650
1651/// F32→F32 linearize step for a transfer function, or `None` if the TF is
1652/// already linear (or Unknown — caller decides how to handle Unknown).
1653fn f32_linearize_step(tf: TransferFunction) -> Option<ConvertStep> {
1654    match tf {
1655        TransferFunction::Linear => None,
1656        TransferFunction::Srgb => Some(ConvertStep::SrgbF32ToLinearF32),
1657        TransferFunction::Bt709 => Some(ConvertStep::Bt709F32ToLinearF32),
1658        TransferFunction::Pq => Some(ConvertStep::PqF32ToLinearF32),
1659        TransferFunction::Hlg => Some(ConvertStep::HlgF32ToLinearF32),
1660        TransferFunction::Gamma22 => Some(ConvertStep::Gamma22F32ToLinearF32),
1661        TransferFunction::Unknown => None,
1662        _ => None,
1663    }
1664}
1665
1666/// F32→F32 OETF step for a transfer function, or `None` if the TF is linear
1667/// (or Unknown).
1668fn f32_encode_step(tf: TransferFunction) -> Option<ConvertStep> {
1669    match tf {
1670        TransferFunction::Linear => None,
1671        TransferFunction::Srgb => Some(ConvertStep::LinearF32ToSrgbF32),
1672        TransferFunction::Bt709 => Some(ConvertStep::LinearF32ToBt709F32),
1673        TransferFunction::Pq => Some(ConvertStep::LinearF32ToPqF32),
1674        TransferFunction::Hlg => Some(ConvertStep::LinearF32ToHlgF32),
1675        TransferFunction::Gamma22 => Some(ConvertStep::LinearF32ToGamma22F32),
1676        TransferFunction::Unknown => None,
1677        _ => None,
1678    }
1679}
1680
1681/// F32→F32 TF-change steps: linearize (if not already linear) then encode
1682/// (if target is not linear).
1683///
1684/// Returns empty when `from == to`, or when either side is `Unknown` — when
1685/// one side's TF is unknown we can't mechanically compute a correct
1686/// conversion, so we preserve bytes as-is. Addressing the Unknown ambiguity
1687/// via explicit opt-in API is tracked as issue #19 `[C]`/`[D]` (deprecate-and-add).
1688fn f32_tf_pair_steps(from: TransferFunction, to: TransferFunction) -> Vec<ConvertStep> {
1689    if from == to || from == TransferFunction::Unknown || to == TransferFunction::Unknown {
1690        return Vec::new();
1691    }
1692    let mut steps = Vec::with_capacity(2);
1693    if let Some(s) = f32_linearize_step(from) {
1694        steps.push(s);
1695    }
1696    if let Some(s) = f32_encode_step(to) {
1697        steps.push(s);
1698    }
1699    steps
1700}
1701
1702/// Depth conversion step into F32 for any non-F32 channel type (U8, U16, F16).
1703/// Panics for F32 (caller must check); CMYK is rejected upstream by
1704/// [`requires_cms`] before any plan steps are picked.
1705fn to_f32_step(ct: ChannelType) -> ConvertStep {
1706    match ct {
1707        ChannelType::U8 => ConvertStep::NaiveU8ToF32,
1708        ChannelType::U16 => ConvertStep::U16ToF32,
1709        ChannelType::F16 => ConvertStep::F16ToF32,
1710        _ => unreachable!("to_f32_step called with F32 or unsupported channel type"),
1711    }
1712}
1713
1714/// F32→depth step for any non-F32 channel type.
1715fn f32_to_depth_step(ct: ChannelType) -> ConvertStep {
1716    match ct {
1717        ChannelType::U8 => ConvertStep::NaiveF32ToU8,
1718        ChannelType::U16 => ConvertStep::F32ToU16,
1719        ChannelType::F16 => ConvertStep::F32ToF16,
1720        _ => unreachable!("f32_to_depth_step called with F32 or unsupported channel type"),
1721    }
1722}
1723
1724/// Determine the depth conversion step(s), considering transfer functions.
1725///
1726/// Returns one or more steps. Multi-step conversions route through an F32
1727/// linear intermediate (e.g. PQ U16 → sRGB U8 goes PQ U16 → Linear F32 →
1728/// sRGB U8), and same-depth integer TF changes route through an F32 linear
1729/// intermediate too: passing integer bytes through unchanged under a new
1730/// TF label produces wrong pixels.
1731fn depth_steps(
1732    from: ChannelType,
1733    to: ChannelType,
1734    from_tf: TransferFunction,
1735    to_tf: TransferFunction,
1736) -> Result<Vec<ConvertStep>, ConvertError> {
1737    if from == to && from_tf == to_tf {
1738        return Ok(Vec::new());
1739    }
1740
1741    // Same depth, F32: apply EOTF/OETF in place.
1742    if from == to && from == ChannelType::F32 {
1743        return Ok(f32_tf_pair_steps(from_tf, to_tf));
1744    }
1745
1746    // Same depth, non-F32 (U8/U16/F16): TF change requires re-encoding. Route
1747    // through F32 linear intermediate — passing bytes through labeled as a
1748    // different TF produces wrong pixels.
1749    //
1750    // Exception: if either TF is Unknown, we don't know the correct conversion.
1751    // Preserve bytes exactly (no F32 round-trip — that would introduce U8/U16
1752    // rounding error for no semantic benefit). Addressed properly by issue
1753    // #19 [C]/[D] via opt-in deprecate-and-add.
1754    if from == to && from != ChannelType::F32 {
1755        if from_tf == TransferFunction::Unknown || to_tf == TransferFunction::Unknown {
1756            return Ok(Vec::new());
1757        }
1758        let mut steps = Vec::with_capacity(4);
1759        steps.push(to_f32_step(from));
1760        steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1761        steps.push(f32_to_depth_step(to));
1762        return Ok(steps);
1763    }
1764
1765    match (from, to) {
1766        (ChannelType::U8, ChannelType::F32) => {
1767            // Fused sRGB EOTF kernel — sRGB only. BT.709 uses a different EOTF
1768            // (~17% linear-light error at mid-gray if we routed it through the
1769            // sRGB kernel) and must compose through the F32 BT.709 EOTF step.
1770            if from_tf == TransferFunction::Srgb && to_tf == TransferFunction::Linear {
1771                Ok(vec![ConvertStep::SrgbU8ToLinearF32])
1772            } else if from_tf == to_tf {
1773                Ok(vec![ConvertStep::NaiveU8ToF32])
1774            } else {
1775                // Cross-depth + cross-TF: linearize/encode after the U8→F32 scale.
1776                // Previously dropped the TF math and returned bytes labeled with
1777                // the target TF — silent wrong pixels for any TF pair other than
1778                // {Srgb,Bt709}→Linear.
1779                let mut steps = Vec::with_capacity(3);
1780                steps.push(ConvertStep::NaiveU8ToF32);
1781                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1782                Ok(steps)
1783            }
1784        }
1785        (ChannelType::F32, ChannelType::U8) => {
1786            // Fused sRGB OETF kernel — sRGB only (same reason as above).
1787            if from_tf == TransferFunction::Linear && to_tf == TransferFunction::Srgb {
1788                Ok(vec![ConvertStep::LinearF32ToSrgbU8])
1789            } else if from_tf == to_tf {
1790                Ok(vec![ConvertStep::NaiveF32ToU8])
1791            } else {
1792                // Linearize/encode in F32 first, then compress to U8.
1793                let mut steps = f32_tf_pair_steps(from_tf, to_tf);
1794                steps.push(ConvertStep::NaiveF32ToU8);
1795                Ok(steps)
1796            }
1797        }
1798        (ChannelType::U16, ChannelType::F32) => {
1799            // PQ/HLG U16 → Linear F32: apply EOTF during conversion.
1800            match (from_tf, to_tf) {
1801                (TransferFunction::Pq, TransferFunction::Linear) => {
1802                    Ok(vec![ConvertStep::PqU16ToLinearF32])
1803                }
1804                (TransferFunction::Hlg, TransferFunction::Linear) => {
1805                    Ok(vec![ConvertStep::HlgU16ToLinearF32])
1806                }
1807                (a, b) if a == b => Ok(vec![ConvertStep::U16ToF32]),
1808                _ => {
1809                    let mut steps = Vec::with_capacity(3);
1810                    steps.push(ConvertStep::U16ToF32);
1811                    steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1812                    Ok(steps)
1813                }
1814            }
1815        }
1816        (ChannelType::F32, ChannelType::U16) => {
1817            // Linear F32 → PQ/HLG U16: apply OETF during conversion.
1818            match (from_tf, to_tf) {
1819                (TransferFunction::Linear, TransferFunction::Pq) => {
1820                    Ok(vec![ConvertStep::LinearF32ToPqU16])
1821                }
1822                (TransferFunction::Linear, TransferFunction::Hlg) => {
1823                    Ok(vec![ConvertStep::LinearF32ToHlgU16])
1824                }
1825                (a, b) if a == b => Ok(vec![ConvertStep::F32ToU16]),
1826                _ => {
1827                    let mut steps = f32_tf_pair_steps(from_tf, to_tf);
1828                    steps.push(ConvertStep::F32ToU16);
1829                    Ok(steps)
1830                }
1831            }
1832        }
1833        (ChannelType::U16, ChannelType::U8) => {
1834            // HDR U16 → SDR U8: go through linear F32 with proper EOTF → OETF.
1835            if from_tf == TransferFunction::Pq && to_tf == TransferFunction::Srgb {
1836                Ok(vec![
1837                    ConvertStep::PqU16ToLinearF32,
1838                    ConvertStep::LinearF32ToSrgbU8,
1839                ])
1840            } else if from_tf == TransferFunction::Hlg && to_tf == TransferFunction::Srgb {
1841                Ok(vec![
1842                    ConvertStep::HlgU16ToLinearF32,
1843                    ConvertStep::LinearF32ToSrgbU8,
1844                ])
1845            } else if from_tf == to_tf {
1846                Ok(vec![ConvertStep::U16ToU8])
1847            } else {
1848                let mut steps = Vec::with_capacity(4);
1849                steps.push(ConvertStep::U16ToF32);
1850                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1851                steps.push(ConvertStep::NaiveF32ToU8);
1852                Ok(steps)
1853            }
1854        }
1855        (ChannelType::U8, ChannelType::U16) => {
1856            if from_tf == to_tf {
1857                Ok(vec![ConvertStep::U8ToU16])
1858            } else {
1859                let mut steps = Vec::with_capacity(4);
1860                steps.push(ConvertStep::NaiveU8ToF32);
1861                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1862                steps.push(ConvertStep::F32ToU16);
1863                Ok(steps)
1864            }
1865        }
1866        // F16 paths route through F32. No fused TF kernels yet — these are
1867        // optimization targets for a future pass.
1868        (ChannelType::F16, ChannelType::F32) => {
1869            let mut steps = Vec::with_capacity(3);
1870            steps.push(ConvertStep::F16ToF32);
1871            if from_tf != to_tf {
1872                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1873            }
1874            Ok(steps)
1875        }
1876        (ChannelType::F32, ChannelType::F16) => {
1877            let mut steps = Vec::with_capacity(3);
1878            if from_tf != to_tf {
1879                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1880            }
1881            steps.push(ConvertStep::F32ToF16);
1882            Ok(steps)
1883        }
1884        (ChannelType::F16, ChannelType::U8) => {
1885            let mut steps = Vec::with_capacity(4);
1886            steps.push(ConvertStep::F16ToF32);
1887            if from_tf == TransferFunction::Linear && to_tf == TransferFunction::Srgb {
1888                steps.push(ConvertStep::LinearF32ToSrgbU8);
1889            } else if from_tf == to_tf {
1890                steps.push(ConvertStep::NaiveF32ToU8);
1891            } else {
1892                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1893                steps.push(ConvertStep::NaiveF32ToU8);
1894            }
1895            Ok(steps)
1896        }
1897        (ChannelType::U8, ChannelType::F16) => {
1898            let mut steps = Vec::with_capacity(4);
1899            if from_tf == TransferFunction::Srgb && to_tf == TransferFunction::Linear {
1900                steps.push(ConvertStep::SrgbU8ToLinearF32);
1901            } else if from_tf == to_tf {
1902                steps.push(ConvertStep::NaiveU8ToF32);
1903            } else {
1904                steps.push(ConvertStep::NaiveU8ToF32);
1905                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1906            }
1907            steps.push(ConvertStep::F32ToF16);
1908            Ok(steps)
1909        }
1910        (ChannelType::F16, ChannelType::U16) => {
1911            let mut steps = Vec::with_capacity(4);
1912            steps.push(ConvertStep::F16ToF32);
1913            if from_tf == TransferFunction::Linear && to_tf == TransferFunction::Pq {
1914                steps.push(ConvertStep::LinearF32ToPqU16);
1915            } else if from_tf == TransferFunction::Linear && to_tf == TransferFunction::Hlg {
1916                steps.push(ConvertStep::LinearF32ToHlgU16);
1917            } else if from_tf == to_tf {
1918                steps.push(ConvertStep::F32ToU16);
1919            } else {
1920                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1921                steps.push(ConvertStep::F32ToU16);
1922            }
1923            Ok(steps)
1924        }
1925        (ChannelType::U16, ChannelType::F16) => {
1926            let mut steps = Vec::with_capacity(4);
1927            if from_tf == TransferFunction::Pq && to_tf == TransferFunction::Linear {
1928                steps.push(ConvertStep::PqU16ToLinearF32);
1929            } else if from_tf == TransferFunction::Hlg && to_tf == TransferFunction::Linear {
1930                steps.push(ConvertStep::HlgU16ToLinearF32);
1931            } else if from_tf == to_tf {
1932                steps.push(ConvertStep::U16ToF32);
1933            } else {
1934                steps.push(ConvertStep::U16ToF32);
1935                steps.extend(f32_tf_pair_steps(from_tf, to_tf));
1936            }
1937            steps.push(ConvertStep::F32ToF16);
1938            Ok(steps)
1939        }
1940        _ => Err(ConvertError::NoPath {
1941            from: PixelDescriptor::new(from, ChannelLayout::Rgb, None, from_tf),
1942            to: PixelDescriptor::new(to, ChannelLayout::Rgb, None, to_tf),
1943        }),
1944    }
1945}
1946
1947// ---------------------------------------------------------------------------
1948// Row conversion kernels
1949// ---------------------------------------------------------------------------
1950
1951/// Pre-allocated scratch buffer for multi-step row conversions.
1952///
1953/// Eliminates per-row heap allocation by reusing two ping-pong halves
1954/// of a single buffer across calls. Create once per [`ConvertPlan`],
1955/// then pass to `convert_row_buffered` for each row.
1956pub(crate) struct ConvertScratch {
1957    /// Single allocation split into two halves via `split_at_mut`.
1958    /// Stored as `Vec<u32>` to guarantee 4-byte alignment, which lets
1959    /// garb and bytemuck use fast aligned paths instead of unaligned fallbacks.
1960    buf: Vec<u32>,
1961    /// Row-persistent scratch for the HDR tone-map kernels (RGB strip +
1962    /// cached `SoftCompress` gamut LUT). Empty placeholder without the
1963    /// `hdr-experimental` feature.
1964    hdr: convert_kernels::HdrKernelScratch,
1965}
1966
1967impl ConvertScratch {
1968    /// Create empty scratch (buffer grows on first use).
1969    pub(crate) fn new() -> Self {
1970        Self {
1971            buf: Vec::new(),
1972            hdr: convert_kernels::HdrKernelScratch::default(),
1973        }
1974    }
1975
1976    /// Ensure the buffer is large enough for two halves of the max
1977    /// intermediate format at the given width.
1978    fn ensure_capacity(&mut self, plan: &ConvertPlan, width: u32) {
1979        let half_bytes = (width as usize) * plan.max_intermediate_bpp();
1980        let total_u32 = (half_bytes * 2).div_ceil(4);
1981        if self.buf.len() < total_u32 {
1982            self.buf.resize(total_u32, 0);
1983        }
1984    }
1985}
1986
1987impl core::fmt::Debug for ConvertScratch {
1988    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1989        f.debug_struct("ConvertScratch")
1990            .field("capacity", &self.buf.capacity())
1991            .finish()
1992    }
1993}
1994
1995/// Convert one row of `width` pixels using a pre-computed plan.
1996///
1997/// `src` and `dst` must be sized for `width` pixels in their respective formats.
1998/// For multi-step plans, an internal scratch buffer is allocated per call.
1999/// Prefer [`RowConverter`](crate::RowConverter) in hot loops (reuses scratch buffers).
2000pub fn convert_row(plan: &ConvertPlan, src: &[u8], dst: &mut [u8], width: u32) {
2001    // Allocating fallback for one-off calls: the scratch starts empty and
2002    // only grows if the plan actually needs it (multi-step ping-pong or an
2003    // HDR tone-map kernel); identity and other single-step plans stay
2004    // allocation-free.
2005    let mut scratch = ConvertScratch::new();
2006    convert_row_buffered(plan, src, dst, width, &mut scratch);
2007}
2008
2009/// Convert one row of `width` pixels, reusing pre-allocated scratch buffers.
2010///
2011/// For multi-step plans this avoids per-row heap allocation by ping-ponging
2012/// between two halves of a scratch buffer. Single-step plans bypass scratch.
2013pub(crate) fn convert_row_buffered(
2014    plan: &ConvertPlan,
2015    src: &[u8],
2016    dst: &mut [u8],
2017    width: u32,
2018    scratch: &mut ConvertScratch,
2019) {
2020    if plan.is_identity() {
2021        let len = min(src.len(), dst.len());
2022        dst[..len].copy_from_slice(&src[..len]);
2023        return;
2024    }
2025
2026    if plan.steps.len() == 1 {
2027        apply_step_u8(
2028            &plan.steps[0],
2029            src,
2030            dst,
2031            width,
2032            plan.from,
2033            plan.to,
2034            plan.pq_anchor_scale,
2035            &mut scratch.hdr,
2036        );
2037        return;
2038    }
2039
2040    scratch.ensure_capacity(plan, width);
2041
2042    // Destructure so the ping-pong halves and the HDR kernel scratch are
2043    // disjoint mutable borrows across the step loop.
2044    let ConvertScratch { buf, hdr } = scratch;
2045    let buf_bytes: &mut [u8] = bytemuck::cast_slice_mut(buf.as_mut_slice());
2046    let half = buf_bytes.len() / 2;
2047    let (buf_a, buf_b) = buf_bytes.split_at_mut(half);
2048
2049    let num_steps = plan.steps.len();
2050    let mut current_desc = plan.from;
2051
2052    for (i, step) in plan.steps.iter().enumerate() {
2053        let is_last = i == num_steps - 1;
2054        let next_desc = if is_last {
2055            plan.to
2056        } else {
2057            intermediate_desc(current_desc, step)
2058        };
2059
2060        let next_len = (width as usize) * next_desc.bytes_per_pixel();
2061        let curr_len = (width as usize) * current_desc.bytes_per_pixel();
2062
2063        // Ping-pong: even steps read src/buf_b and write buf_a;
2064        // odd steps read buf_a and write buf_b. Each branch only
2065        // borrows each half in one mode, satisfying the borrow checker.
2066        if i % 2 == 0 {
2067            let input = if i == 0 { src } else { &buf_b[..curr_len] };
2068            if is_last {
2069                apply_step_u8(
2070                    step,
2071                    input,
2072                    dst,
2073                    width,
2074                    current_desc,
2075                    next_desc,
2076                    plan.pq_anchor_scale,
2077                    &mut *hdr,
2078                );
2079            } else {
2080                apply_step_u8(
2081                    step,
2082                    input,
2083                    &mut buf_a[..next_len],
2084                    width,
2085                    current_desc,
2086                    next_desc,
2087                    plan.pq_anchor_scale,
2088                    &mut *hdr,
2089                );
2090            }
2091        } else {
2092            let input = &buf_a[..curr_len];
2093            if is_last {
2094                apply_step_u8(
2095                    step,
2096                    input,
2097                    dst,
2098                    width,
2099                    current_desc,
2100                    next_desc,
2101                    plan.pq_anchor_scale,
2102                    &mut *hdr,
2103                );
2104            } else {
2105                apply_step_u8(
2106                    step,
2107                    input,
2108                    &mut buf_b[..next_len],
2109                    width,
2110                    current_desc,
2111                    next_desc,
2112                    plan.pq_anchor_scale,
2113                    &mut *hdr,
2114                );
2115            }
2116        }
2117
2118        current_desc = next_desc;
2119    }
2120}
2121
2122/// Check if two steps are inverses that cancel each other.
2123/// Collapse `[SrgbU8ToLinearF32, GamutMatrix*F32(m), LinearF32ToSrgbU8]`
2124/// into a single fused matlut step. Mutates in place.
2125fn fuse_matlut_patterns(steps: &mut Vec<ConvertStep>) {
2126    let mut i = 0;
2127    while i + 2 < steps.len() {
2128        let rewrite = match (&steps[i], &steps[i + 1], &steps[i + 2]) {
2129            (
2130                ConvertStep::SrgbU8ToLinearF32,
2131                ConvertStep::GamutMatrixRgbF32(m),
2132                ConvertStep::LinearF32ToSrgbU8,
2133            ) => Some(ConvertStep::Fused {
2134                kind: FusedKind::SrgbU8GamutRgb,
2135                matrix: *m,
2136            }),
2137            (
2138                ConvertStep::SrgbU8ToLinearF32,
2139                ConvertStep::GamutMatrixRgbaF32(m),
2140                ConvertStep::LinearF32ToSrgbU8,
2141            ) => Some(ConvertStep::Fused {
2142                kind: FusedKind::SrgbU8GamutRgba,
2143                matrix: *m,
2144            }),
2145            _ => None,
2146        };
2147        if let Some(fused) = rewrite {
2148            steps[i] = fused;
2149            steps.drain(i + 1..i + 3);
2150            continue;
2151        }
2152        i += 1;
2153    }
2154}
2155
2156fn are_inverse(a: &ConvertStep, b: &ConvertStep) -> bool {
2157    matches!(
2158        (a, b),
2159        // Self-inverse
2160        (ConvertStep::SwizzleBgraRgba, ConvertStep::SwizzleBgraRgba)
2161        // Layout inverses (lossless for opaque data)
2162        | (ConvertStep::AddAlpha, ConvertStep::DropAlpha)
2163        // Transfer function f32↔f32 (exact inverses in float)
2164        | (ConvertStep::SrgbF32ToLinearF32, ConvertStep::LinearF32ToSrgbF32)
2165        | (ConvertStep::LinearF32ToSrgbF32, ConvertStep::SrgbF32ToLinearF32)
2166        | (ConvertStep::PqF32ToLinearF32, ConvertStep::LinearF32ToPqF32)
2167        | (ConvertStep::LinearF32ToPqF32, ConvertStep::PqF32ToLinearF32)
2168        | (ConvertStep::HlgF32ToLinearF32, ConvertStep::LinearF32ToHlgF32)
2169        | (ConvertStep::LinearF32ToHlgF32, ConvertStep::HlgF32ToLinearF32)
2170        | (ConvertStep::Bt709F32ToLinearF32, ConvertStep::LinearF32ToBt709F32)
2171        | (ConvertStep::LinearF32ToBt709F32, ConvertStep::Bt709F32ToLinearF32)
2172        | (ConvertStep::Gamma22F32ToLinearF32, ConvertStep::LinearF32ToGamma22F32)
2173        | (ConvertStep::LinearF32ToGamma22F32, ConvertStep::Gamma22F32ToLinearF32)
2174        // Alpha mode (exact inverses in float)
2175        | (ConvertStep::StraightToPremul, ConvertStep::PremulToStraight)
2176        | (ConvertStep::PremulToStraight, ConvertStep::StraightToPremul)
2177        // Color model (exact inverses in float)
2178        | (ConvertStep::LinearRgbToOklab, ConvertStep::OklabToLinearRgb)
2179        | (ConvertStep::OklabToLinearRgb, ConvertStep::LinearRgbToOklab)
2180        | (ConvertStep::LinearRgbaToOklaba, ConvertStep::OklabaToLinearRgba)
2181        | (ConvertStep::OklabaToLinearRgba, ConvertStep::LinearRgbaToOklaba)
2182        // Cross-depth pairs (near-lossless for same depth class)
2183        | (ConvertStep::NaiveU8ToF32, ConvertStep::NaiveF32ToU8)
2184        | (ConvertStep::NaiveF32ToU8, ConvertStep::NaiveU8ToF32)
2185        | (ConvertStep::U8ToU16, ConvertStep::U16ToU8)
2186        | (ConvertStep::U16ToU8, ConvertStep::U8ToU16)
2187        | (ConvertStep::U16ToF32, ConvertStep::F32ToU16)
2188        | (ConvertStep::F32ToU16, ConvertStep::U16ToF32)
2189        | (ConvertStep::F16ToF32, ConvertStep::F32ToF16)
2190        | (ConvertStep::F32ToF16, ConvertStep::F16ToF32)
2191        // Cross-depth with transfer (near-lossless roundtrip)
2192        | (ConvertStep::SrgbU8ToLinearF32, ConvertStep::LinearF32ToSrgbU8)
2193        | (ConvertStep::LinearF32ToSrgbU8, ConvertStep::SrgbU8ToLinearF32)
2194        | (ConvertStep::PqU16ToLinearF32, ConvertStep::LinearF32ToPqU16)
2195        | (ConvertStep::LinearF32ToPqU16, ConvertStep::PqU16ToLinearF32)
2196        | (ConvertStep::HlgU16ToLinearF32, ConvertStep::LinearF32ToHlgU16)
2197        | (ConvertStep::LinearF32ToHlgU16, ConvertStep::HlgU16ToLinearF32)
2198        // Extended-range sRGB f32 pairs
2199        | (ConvertStep::SrgbF32ToLinearF32Extended, ConvertStep::LinearF32ToSrgbF32Extended)
2200        | (ConvertStep::LinearF32ToSrgbF32Extended, ConvertStep::SrgbF32ToLinearF32Extended)
2201    )
2202}
2203
2204/// Compute the descriptor after applying one step.
2205fn intermediate_desc(current: PixelDescriptor, step: &ConvertStep) -> PixelDescriptor {
2206    match step {
2207        ConvertStep::Identity => current,
2208        ConvertStep::SwizzleBgraRgba => {
2209            let new_layout = match current.layout() {
2210                ChannelLayout::Bgra => ChannelLayout::Rgba,
2211                ChannelLayout::Rgba => ChannelLayout::Bgra,
2212                other => other,
2213            };
2214            PixelDescriptor::new(
2215                current.channel_type(),
2216                new_layout,
2217                current.alpha(),
2218                current.transfer(),
2219            )
2220        }
2221        ConvertStep::AddAlpha => PixelDescriptor::new(
2222            current.channel_type(),
2223            ChannelLayout::Rgba,
2224            Some(AlphaMode::Straight),
2225            current.transfer(),
2226        ),
2227        ConvertStep::RgbToBgra => PixelDescriptor::new(
2228            current.channel_type(),
2229            ChannelLayout::Bgra,
2230            Some(AlphaMode::Straight),
2231            current.transfer(),
2232        ),
2233        ConvertStep::DropAlpha | ConvertStep::MatteComposite { .. } => PixelDescriptor::new(
2234            current.channel_type(),
2235            ChannelLayout::Rgb,
2236            None,
2237            current.transfer(),
2238        ),
2239        ConvertStep::GrayToRgb => PixelDescriptor::new(
2240            current.channel_type(),
2241            ChannelLayout::Rgb,
2242            None,
2243            current.transfer(),
2244        ),
2245        ConvertStep::GrayToRgba => PixelDescriptor::new(
2246            current.channel_type(),
2247            ChannelLayout::Rgba,
2248            Some(AlphaMode::Straight),
2249            current.transfer(),
2250        ),
2251        ConvertStep::RgbToGray { .. } | ConvertStep::RgbaToGray { .. } => PixelDescriptor::new(
2252            current.channel_type(),
2253            ChannelLayout::Gray,
2254            None,
2255            current.transfer(),
2256        ),
2257        ConvertStep::GrayAlphaToRgba => PixelDescriptor::new(
2258            current.channel_type(),
2259            ChannelLayout::Rgba,
2260            current.alpha(),
2261            current.transfer(),
2262        ),
2263        ConvertStep::GrayAlphaToRgb => PixelDescriptor::new(
2264            current.channel_type(),
2265            ChannelLayout::Rgb,
2266            None,
2267            current.transfer(),
2268        ),
2269        ConvertStep::GrayToGrayAlpha => PixelDescriptor::new(
2270            current.channel_type(),
2271            ChannelLayout::GrayAlpha,
2272            Some(AlphaMode::Straight),
2273            current.transfer(),
2274        ),
2275        ConvertStep::GrayAlphaToGray => PixelDescriptor::new(
2276            current.channel_type(),
2277            ChannelLayout::Gray,
2278            None,
2279            current.transfer(),
2280        ),
2281        ConvertStep::SrgbU8ToLinearF32
2282        | ConvertStep::NaiveU8ToF32
2283        | ConvertStep::U16ToF32
2284        | ConvertStep::PqU16ToLinearF32
2285        | ConvertStep::HlgU16ToLinearF32
2286        | ConvertStep::PqF32ToLinearF32
2287        | ConvertStep::HlgF32ToLinearF32
2288        | ConvertStep::SrgbF32ToLinearF32
2289        | ConvertStep::SrgbF32ToLinearF32Extended
2290        | ConvertStep::Bt709F32ToLinearF32
2291        | ConvertStep::Gamma22F32ToLinearF32 => PixelDescriptor::new(
2292            ChannelType::F32,
2293            current.layout(),
2294            current.alpha(),
2295            TransferFunction::Linear,
2296        ),
2297        ConvertStep::LinearF32ToSrgbU8 | ConvertStep::NaiveF32ToU8 | ConvertStep::U16ToU8 => {
2298            PixelDescriptor::new(
2299                ChannelType::U8,
2300                current.layout(),
2301                current.alpha(),
2302                TransferFunction::Srgb,
2303            )
2304        }
2305        ConvertStep::U8ToU16 => PixelDescriptor::new(
2306            ChannelType::U16,
2307            current.layout(),
2308            current.alpha(),
2309            current.transfer(),
2310        ),
2311        ConvertStep::F32ToU16 | ConvertStep::LinearF32ToPqU16 | ConvertStep::LinearF32ToHlgU16 => {
2312            let tf = match step {
2313                ConvertStep::LinearF32ToPqU16 => TransferFunction::Pq,
2314                ConvertStep::LinearF32ToHlgU16 => TransferFunction::Hlg,
2315                _ => current.transfer(),
2316            };
2317            PixelDescriptor::new(ChannelType::U16, current.layout(), current.alpha(), tf)
2318        }
2319        ConvertStep::LinearF32ToPqF32 => PixelDescriptor::new(
2320            ChannelType::F32,
2321            current.layout(),
2322            current.alpha(),
2323            TransferFunction::Pq,
2324        ),
2325        ConvertStep::LinearF32ToHlgF32 => PixelDescriptor::new(
2326            ChannelType::F32,
2327            current.layout(),
2328            current.alpha(),
2329            TransferFunction::Hlg,
2330        ),
2331        ConvertStep::LinearF32ToSrgbF32 | ConvertStep::LinearF32ToSrgbF32Extended => {
2332            PixelDescriptor::new(
2333                ChannelType::F32,
2334                current.layout(),
2335                current.alpha(),
2336                TransferFunction::Srgb,
2337            )
2338        }
2339        ConvertStep::LinearF32ToBt709F32 => PixelDescriptor::new(
2340            ChannelType::F32,
2341            current.layout(),
2342            current.alpha(),
2343            TransferFunction::Bt709,
2344        ),
2345        ConvertStep::LinearF32ToGamma22F32 => PixelDescriptor::new(
2346            ChannelType::F32,
2347            current.layout(),
2348            current.alpha(),
2349            TransferFunction::Gamma22,
2350        ),
2351        ConvertStep::StraightToPremul => PixelDescriptor::new(
2352            current.channel_type(),
2353            current.layout(),
2354            Some(AlphaMode::Premultiplied),
2355            current.transfer(),
2356        ),
2357        ConvertStep::PremulToStraight => PixelDescriptor::new(
2358            current.channel_type(),
2359            current.layout(),
2360            Some(AlphaMode::Straight),
2361            current.transfer(),
2362        ),
2363        ConvertStep::LinearRgbToOklab => PixelDescriptor::new(
2364            ChannelType::F32,
2365            ChannelLayout::Oklab,
2366            None,
2367            TransferFunction::Unknown,
2368        )
2369        .with_primaries(current.primaries),
2370        ConvertStep::OklabToLinearRgb => PixelDescriptor::new(
2371            ChannelType::F32,
2372            ChannelLayout::Rgb,
2373            None,
2374            TransferFunction::Linear,
2375        )
2376        .with_primaries(current.primaries),
2377        ConvertStep::LinearRgbaToOklaba => PixelDescriptor::new(
2378            ChannelType::F32,
2379            ChannelLayout::OklabA,
2380            Some(AlphaMode::Straight),
2381            TransferFunction::Unknown,
2382        )
2383        .with_primaries(current.primaries),
2384        ConvertStep::OklabaToLinearRgba => PixelDescriptor::new(
2385            ChannelType::F32,
2386            ChannelLayout::Rgba,
2387            current.alpha(),
2388            TransferFunction::Linear,
2389        )
2390        .with_primaries(current.primaries),
2391
2392        // Gamut matrix: same depth/layout/TF, but primaries change.
2393        // The actual target primaries are embedded in the matrix, not tracked
2394        // here — we mark them as Unknown since the step doesn't carry that info.
2395        // The final plan.to descriptor has the correct primaries.
2396        ConvertStep::GamutMatrixRgbF32(_) => PixelDescriptor::new(
2397            ChannelType::F32,
2398            current.layout(),
2399            current.alpha(),
2400            TransferFunction::Linear,
2401        ),
2402        ConvertStep::GamutMatrixRgbaF32(_) => PixelDescriptor::new(
2403            ChannelType::F32,
2404            current.layout(),
2405            current.alpha(),
2406            TransferFunction::Linear,
2407        ),
2408        // Fused steps: shape depends on FusedKind.
2409        ConvertStep::Fused { kind, .. } => {
2410            let (ch_type, transfer) = match kind {
2411                // u8 sRGB in, u8 sRGB out (same layout, same alpha).
2412                FusedKind::SrgbU8GamutRgb | FusedKind::SrgbU8GamutRgba => {
2413                    (ChannelType::U8, TransferFunction::Srgb)
2414                }
2415                FusedKind::SrgbU16GamutRgb => (ChannelType::U16, TransferFunction::Srgb),
2416                FusedKind::SrgbU8ToLinearF32Rgb => (ChannelType::F32, TransferFunction::Linear),
2417                FusedKind::LinearF32ToSrgbU8Rgb => (ChannelType::U8, TransferFunction::Srgb),
2418            };
2419            PixelDescriptor::new(ch_type, current.layout(), current.alpha(), transfer)
2420        }
2421        // F16↔F32 depth-only steps. No TF implication: same TF on both sides.
2422        ConvertStep::F16ToF32 => PixelDescriptor::new(
2423            ChannelType::F32,
2424            current.layout(),
2425            current.alpha(),
2426            current.transfer(),
2427        ),
2428        ConvertStep::F32ToF16 => PixelDescriptor::new(
2429            ChannelType::F16,
2430            current.layout(),
2431            current.alpha(),
2432            current.transfer(),
2433        ),
2434        // HDR steps. Both operate on linear-light F32 RGB and preserve the
2435        // layout/alpha/transfer/depth of the carrier. ToneMapBt2446A operates
2436        // in BT.2020 (planner-enforced); SoftCompressOklch operates in the
2437        // step's stored `primaries`. Neither step changes the descriptor
2438        // shape — they only update pixel values.
2439        #[cfg(feature = "hdr-experimental")]
2440        ConvertStep::ToneMapBt2446A { .. } => current,
2441        #[cfg(feature = "hdr-experimental")]
2442        ConvertStep::SoftCompressOklch { .. } => current,
2443    }
2444}
2445
2446#[path = "convert_kernels.rs"]
2447mod convert_kernels;
2448use convert_kernels::apply_step_u8;
2449pub(crate) use convert_kernels::{hlg_eotf, hlg_oetf, pq_eotf, pq_oetf};
2450
2451#[cfg(all(test, feature = "hdr-experimental"))]
2452mod hdr_plan_tests {
2453    //! Unit tests pinning the HDR-aware `ConvertPlan` against the same math
2454    //! the deleted `HdrToSdr::apply_strip` ran. Keeps the e2e ΔE2000 budget
2455    //! grounded in per-pixel parity rather than only the imazen-26 sample.
2456    use super::*;
2457    use crate::gamut::{apply_matrix_f32, conversion_matrix};
2458    use crate::hdr::{Bt2446A, SoftCompress};
2459    use crate::oklab;
2460
2461    /// Reproduce the strip math of the deleted `HdrToSdr::apply_strip` for
2462    /// a BT.709 (linear) → BT.709 (linear) pipeline at 1000 nit source peak.
2463    fn reference_pipeline(input: [f32; 3]) -> [f32; 3] {
2464        let mut px = [input];
2465        // Scrub.
2466        for c in px[0].iter_mut() {
2467            if !c.is_finite() || *c < 0.0 {
2468                *c = 0.0;
2469            }
2470        }
2471        // Source primaries → BT.2020.
2472        let m_src = conversion_matrix(ColorPrimaries::Bt709, ColorPrimaries::Bt2020).unwrap();
2473        for p in px.iter_mut() {
2474            apply_matrix_f32(p, &m_src);
2475        }
2476        // BT.2446-A curve in BT.2020.
2477        Bt2446A::new(1000.0, 100.0).map_strip_simd(&mut px);
2478        // BT.2020 → target primaries.
2479        let m_dst = conversion_matrix(ColorPrimaries::Bt2020, ColorPrimaries::Bt709).unwrap();
2480        for p in px.iter_mut() {
2481            apply_matrix_f32(p, &m_dst);
2482        }
2483        // OKLch soft compress in target primaries.
2484        let m1 = oklab::rgb_to_lms_matrix(ColorPrimaries::Bt709).unwrap();
2485        let m1_inv = oklab::lms_to_rgb_matrix(ColorPrimaries::Bt709).unwrap();
2486        let compressor = SoftCompress::from_matrices(&m1, &m1_inv, 0.96);
2487        compressor.apply_strip(&mut px);
2488        // Final clamp.
2489        for c in px[0].iter_mut() {
2490            if !c.is_finite() {
2491                *c = 0.0;
2492            } else {
2493                *c = c.clamp(0.0, 1.0);
2494            }
2495        }
2496        px[0]
2497    }
2498
2499    /// Single-pixel sanity using the same entry the e2e test uses
2500    /// (`PixelBufferHdrConvertExt::convert_to_with_hdr_config`) — pin
2501    /// that the user-facing extension method routes through the same
2502    /// kernels the manual reference uses.
2503    #[test]
2504    fn pixel_buffer_hdr_convert_matches_reference_pipeline() {
2505        use crate::PixelBufferHdrConvertExt;
2506        use zenpixels::PixelBuffer;
2507        let src = PixelDescriptor::new_full(
2508            ChannelType::F32,
2509            ChannelLayout::Rgb,
2510            None,
2511            TransferFunction::Linear,
2512            ColorPrimaries::Bt709,
2513        );
2514        let to = PixelDescriptor::new_full(
2515            ChannelType::F32,
2516            ChannelLayout::Rgb,
2517            None,
2518            TransferFunction::Linear,
2519            ColorPrimaries::Bt709,
2520        );
2521        let hdr = HdrConfig::for_source_peak(1000.0);
2522        let inputs = [
2523            [0.0_f32, 0.0, 0.0],
2524            [0.18, 0.18, 0.18],
2525            [1.0, 1.0, 1.0],
2526            [0.5, 0.3, 0.1],
2527        ];
2528        for inp in inputs {
2529            let expected = reference_pipeline(inp);
2530            let bytes: Vec<u8> = bytemuck::cast_slice(&inp).to_vec();
2531            let buf = PixelBuffer::from_vec(bytes, 1, 1, src).unwrap();
2532            let out = buf.convert_to_with_hdr_config(to, hdr).expect("convert");
2533            let out_bytes = out.copy_to_contiguous_bytes();
2534            let got: &[f32] = bytemuck::cast_slice(&out_bytes);
2535            for k in 0..3 {
2536                let diff = (expected[k] - got[k]).abs();
2537                assert!(
2538                    diff < 5e-4,
2539                    "ext channel {k} for input {inp:?}: expected {} vs got {} (diff {})",
2540                    expected[k],
2541                    got[k],
2542                    diff,
2543                );
2544            }
2545        }
2546    }
2547
2548    #[test]
2549    fn hdr_plan_matches_reference_pipeline_for_bt709_linear_targets() {
2550        let src = PixelDescriptor::new_full(
2551            ChannelType::F32,
2552            ChannelLayout::Rgb,
2553            None,
2554            TransferFunction::Linear,
2555            ColorPrimaries::Bt709,
2556        );
2557        let to = PixelDescriptor::new_full(
2558            ChannelType::F32,
2559            ChannelLayout::Rgb,
2560            None,
2561            TransferFunction::Linear,
2562            ColorPrimaries::Bt709,
2563        );
2564        let hdr = HdrConfig::for_source_peak(1000.0);
2565        let plan = ConvertPlan::new_with_hdr_config(src, to, hdr).expect("plan");
2566        let inputs = [
2567            [0.0_f32, 0.0, 0.0],
2568            [0.18, 0.18, 0.18],
2569            [1.0, 1.0, 1.0],
2570            [0.5, 0.3, 0.1],
2571            [0.9, 0.1, 0.05],
2572        ];
2573        for inp in inputs {
2574            let expected = reference_pipeline(inp);
2575            let bytes: Vec<u8> = bytemuck::cast_slice(&inp).to_vec();
2576            let mut out = vec![0u8; 12];
2577            convert_row(&plan, &bytes, &mut out, 1);
2578            let got_f: &[f32] = bytemuck::cast_slice(&out);
2579            for k in 0..3 {
2580                let diff = (expected[k] - got_f[k]).abs();
2581                assert!(
2582                    diff < 5e-4,
2583                    "channel {k} for input {inp:?}: expected {} vs got {} (diff {})",
2584                    expected[k],
2585                    got_f[k],
2586                    diff,
2587                );
2588            }
2589        }
2590    }
2591}