cranpose_render_wgpu/

render.rs

use std::{
    borrow::Cow,
    cell::Cell,
    hash::{Hash, Hasher},
    rc::Rc,
    sync::{Arc, mpsc},
    time::Duration,
};

use bytemuck::{Pod, Zeroable};
use cranpose_core::{
    NodeId,
    collections::{bounded_lru::BoundedLruCache, map::HashMap},
    hash::default as default_hash,
};
use cranpose_render_common::{
    geometry::blur_reach,
    graph::{DrawCommandId, quad_bounds},
    software_text_raster::{
        SoftwareGlyphAtlasGlyph, SoftwareGlyphAtlasKey, SoftwareGlyphAtlasPlacement,
        SoftwareGlyphAtlasRunGlyph, SoftwareGlyphRasterCache, SoftwareTextFontSet,
        collect_solid_text_atlas_run, rasterize_annotated_text_region,
    },
    text_mask_gamma::TextLuminance,
};
use cranpose_ui_graphics::{
    BlendMode, Color, ColorFilter, FRAGMENT_KIND_ARC, FRAGMENT_KIND_FILL, FRAGMENT_KIND_LINE,
    FRAGMENT_KIND_STROKE, FRAGMENT_KIND_TRAPEZOID, FxHasher, ImageBitmap, ImageSampling, Point,
    RecordLane, RecordSegment, Rect, RenderHash, TileMode,
};
use smallvec::SmallVec;
use web_time::Instant;

use crate::{
    DebugCpuAllocationStats,
    ablation::{Ablation, ShapeAblation},
    arc_trig_fill::ArcTrigFill,
    collect::LayerScene,
    debug_toggles::DebugToggle,
    draw_pass::{
        PassSegment, PassTarget, ResolvedComposite, ResolvedCompositeKind, SourceContent,
        TrackedPass,
    },
    effect_renderer::{CompositeSampleMode, EffectRenderer, RoundedCompositeMask},
    fixed_pipeline::{FixedPipeline, FixedPipelineJob},
    frame::{AdmissionGate, FrameExecutor, StageSideScratch},
    frame_graph::{
        BufferUpload, FrameCommandRecorder, FrameCommandStats, FrameTextureDescriptor,
        FrameUploadAllocators, UniformUpload, UploadAllocatorId, UploadAllocatorSpec, UploadMode,
        WgpuFrameGraph, WgpuFrameGraphExecutor,
    },
    frame_packet::{CancelReason, FramePacket, FrameSceneStorage, PresentOutcome, RenderReturns},
    geometry::{
        DevicePixelBounds, SegmentTransform, anchored_device_rect, axis_aligned_quad_rect,
        canonicalize_device_coordinate, canonicalized_scaled_quad, offscreen_byte_size,
        scaled_quad, snap_delta_for_anchor, snapped_anchor_device_origin, translate_quad,
        translation_stable_anchored_device_pixel_bounds,
    },
    glyph_run::{RunGlyphBox, RunGlyphScratch, RunGlyphs},
    glyph_run_arena::{GlyphRunArena, GlyphRunSpan},
    gpu_stats::{self, gpu_stats_enabled},
    idle_pool::FrameScratch,
    layer_cache::LayerCache,
    lazy_resource::Pending,
    offscreen::{COMPOSITION_BYTES_PER_PIXEL, OffscreenTarget},
    output_conversion::OutputConverter,
    pipeline_compiler::{CompileLane, CompilerSend, PipelineCompilation, PipelineCompiler},
    record_columns::record_vertex_layouts,
    rect_to_quad,
    rrect_shadow::{SHADOW_QUAD_CORNERS, ShadowInstance, create_rrect_shadow_pipeline},
    run_store::{ArenaBinding, PlacementData, RunBufferMode, RunDrawCall, RunStore, StoredTables},
    scene::{
        CompositorScene, DrawOp, DrawOpKind, DrawnText, ImageDraw, RunDraw, ShadowDraw, SnapAnchor,
        TextDraw,
    },
    shaders,
    shape_pipelines::{PipelineDemand, ShapePipelineFactory, ShapePipelines},
    shared_shader::{ShaderCode, SharedShader},
};
const MAX_SHADOW_SURFACE_CACHE_ITEMS: usize = 512;
const MAX_TRANSPARENT_SOURCES: usize = 16;
const MAX_SHADOW_SURFACE_CACHE_BYTES: u64 = 384 * 1024 * 1024;

static SKIP_SHADOWS: DebugToggle = DebugToggle::new("CRANPOSE_SKIP_SHADOWS");

fn skip_shadow_draws() -> bool {
    SKIP_SHADOWS.flag()
}
const MAX_TEXT_IMAGE_CACHE_ITEMS: usize = 1024;
const MAX_TEXT_GLYPH_MASK_CACHE_ITEMS: usize = 8192;
const MAX_TEXT_GLYPH_ATLAS_ITEMS: usize = 8192;
/// The fewest text runs the run cache holds. It holds twice the runs the
/// last frame drew when that is more, so a frame never evicts runs the next
/// one draws again; idle runs leave it after `TEXT_GLYPH_RUN_IDLE_FRAMES`.
const MIN_TEXT_GLYPH_RUN_CACHE_ITEMS: usize = 1024;
const MAX_TEXT_GLYPH_GPU_RUN_CACHE_ITEMS: usize = 1024;
/// Frames a text run may go undrawn before its glyphs and quads are freed,
/// on the CPU and in retained GPU buffers alike. Shorter than the shape
/// store's span: a list scrolling at speed leaves several screens of text
/// behind each second, which at 120 frames held 12 to 19 MB of GPU quads on
/// a scrolling feed, and the CPU runs, bounded only by their count, held
/// another 4.6 MB of placements and quads.
const TEXT_GLYPH_RUN_IDLE_FRAMES: u64 = 30;
/// Frames a retained run's quads stay on the GPU undrawn. Writing a run's
/// quads again costs far less than laying its glyphs out, which the CPU run
/// keeps for [`TEXT_GLYPH_RUN_IDLE_FRAMES`]. On the desktop gauntlet at tier
/// 16, 270 retained runs drew each frame while 30 frames held 650 of them;
/// 8 frames hold 400 and write 25 runs a frame again instead of 16.
const TEXT_GLYPH_GPU_RUN_IDLE_FRAMES: u64 = 8;
/// Text runs with at least this many glyphs keep their quads in retained GPU
/// buffers and draw on their own; shorter ones are written into the frame's
/// shared quads each frame, where consecutive runs share a draw.
const RETAINED_TEXT_GLYPH_RUN_MIN_QUADS: usize = 64;

const TEXT_GLYPH_ATLAS_MIN_SIZE: u32 = 512;
const TEXT_GLYPH_ATLAS_MAX_SIZE: u32 = 4096;
const _: () = assert!(TEXT_GLYPH_ATLAS_MAX_SIZE <= u16::MAX as u32);
const TEXT_GLYPH_ATLAS_PADDING: u32 = 1;
const MAX_TEXT_LINE_INDEX_CACHE_ITEMS: usize = 512;
const MIN_MULTILINE_TEXT_LINES_FOR_CLIPPED_RASTER: usize = 2;

const CACHE_MISS_WARMUP_FRAMES: u8 = 1;
pub(crate) const CLEAR_COLOR: wgpu::Color = wgpu::Color {
    r: cranpose_render_common::FRAME_CLEAR_COLOR[0] as f64,
    g: cranpose_render_common::FRAME_CLEAR_COLOR[1] as f64,
    b: cranpose_render_common::FRAME_CLEAR_COLOR[2] as f64,
    a: cranpose_render_common::FRAME_CLEAR_COLOR[3] as f64,
};
const MAX_TEXTURE_CACHE_ITEMS: usize = 256;
const MAX_IMAGE_TEXTURE_CACHE_BYTES: usize = 256 * 1024 * 1024;
const MAX_EXTERNAL_TEXTURE_BINDINGS: usize = 8;

const DEFAULT_WGPU_RENDER_STAGE_TELEMETRY_THRESHOLD_MS: f64 = 4.0;

fn wgpu_render_stage_telemetry_threshold_ms() -> Option<f64> {
    static THRESHOLD_MS: std::sync::OnceLock<Option<f64>> = std::sync::OnceLock::new();
    *THRESHOLD_MS.get_or_init(|| {
        let explicit =
            crate::debug_toggles::debug_toggle("CRANPOSE_WGPU_RENDER_STAGE_TELEMETRY_MS")
                .and_then(|value| value.parse::<f64>().ok())
                .filter(|value| value.is_finite() && *value >= 0.0);
        explicit.or_else(|| {
            std::env::var_os("CRANPOSE_WGPU_RENDER_STAGE_TELEMETRY")
                .is_some()
                .then_some(DEFAULT_WGPU_RENDER_STAGE_TELEMETRY_THRESHOLD_MS)
        })
    })
}

pub(crate) fn instant_ms(start: Instant, end: Instant) -> f64 {
    end.duration_since(start).as_secs_f64() * 1000.0
}

pub(crate) fn should_log_wgpu_render_stage(start: Instant, end: Instant) -> Option<f64> {
    let threshold_ms = wgpu_render_stage_telemetry_threshold_ms()?;
    let total_ms = instant_ms(start, end);
    (total_ms >= threshold_ms).then_some(total_ms)
}

pub static PRESENTED_FRAMES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);

pub fn frames_presented() -> u64 {
    PRESENTED_FRAMES.load(std::sync::atomic::Ordering::Relaxed)
}

fn text_atlas_fallback_diag_enabled() -> bool {
    cranpose_core::env_flag!("CRANPOSE_TEXT_ATLAS_FALLBACK_DIAG")
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct ShadowSurfaceCacheKey {
    content_hash: u64,
    pixel_size: [u32; 2],
    root_scale_bits: u32,
    blur_radius_bits: u32,
}

struct CachedShadowSurface {
    target: Rc<OffscreenTarget>,
    byte_size: u64,
}

type DeviceRect4 = (f32, f32, f32, f32);
/// A rect of whole target pixels: x, y, width, height.
pub(crate) type TargetRect = (u32, u32, u32, u32);

/// The target pixels a clipped shape variant keeps of `placement`'s records
/// under `viewport`, as a scissor that stands in for its clip test: every
/// pixel whose centre, moved by the viewport's offset, lies inside the
/// device clip, inclusive of its edges, as the shader compares them. `None`
/// without a clip or under a turn, where the variant tests the clip itself.
/// An empty clip keeps an empty scissor.
pub(crate) fn clip_scissor(
    placement: &crate::scene::Placement,
    root_scale: f32,
    viewport: ViewportUniformParams,
) -> Option<TargetRect> {
    if !viewport.transform.is_identity() {
        return None;
    }
    let [x, y, width, height] = crate::run_store::device_clip(placement, root_scale)?;
    let columns = kept_pixels(x, x + width, viewport.offset[0], viewport.width);
    let rows = kept_pixels(y, y + height, viewport.offset[1], viewport.height);
    Some((
        columns.start,
        rows.start,
        columns.end - columns.start,
        rows.end - rows.start,
    ))
}

/// The pixels along one axis whose centre, moved by `offset`, lies within
/// `low..=high`, within `0..extent`: stepped with the shader's own float
/// sums so the edges fall where its comparisons put them.
fn kept_pixels(low: f32, high: f32, offset: f32, extent: u32) -> std::ops::Range<u32> {
    let centre = |pixel: u32| pixel as f32 + 0.5 + offset;
    let guess = |edge: f32| (edge - offset - 0.5).ceil().clamp(0.0, extent as f32) as u32;
    let mut start = guess(low);
    while start > 0 && centre(start - 1) >= low {
        start -= 1;
    }
    while start < extent && centre(start) < low {
        start += 1;
    }
    let mut end = guess(high).max(start);
    while end > start && centre(end - 1) > high {
        end -= 1;
    }
    while end < extent && centre(end) <= high {
        end += 1;
    }
    start..end
}

/// The pixels both scissors keep, `None` when they share none.
pub(crate) fn intersect_target_rects(a: TargetRect, b: TargetRect) -> Option<TargetRect> {
    let left = a.0.max(b.0);
    let top = a.1.max(b.1);
    let right = (a.0 + a.2).min(b.0 + b.2);
    let bottom = (a.1 + a.3).min(b.1 + b.3);
    (right > left && bottom > top).then(|| (left, top, right - left, bottom - top))
}

/// A draw's scissor cut down to the pixels its pass segment may touch;
/// `None` when nothing of it remains.
pub(crate) fn bounded_scissor(
    scissor: (u32, u32, u32, u32),
    bound: Option<(u32, u32, u32, u32)>,
) -> Option<(u32, u32, u32, u32)> {
    let Some((bx, by, bw, bh)) = bound else {
        return Some(scissor);
    };
    let (x, y, width, height) = scissor;
    let left = x.max(bx);
    let top = y.max(by);
    let right = (x + width).min(bx + bw);
    let bottom = (y + height).min(by + bh);
    (right > left && bottom > top).then(|| (left, top, right - left, bottom - top))
}

#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct GlyphExtent {
    left: f32,
    top: f32,
    right: f32,
    bottom: f32,
}

impl GlyphExtent {
    pub(crate) const EMPTY: Self = Self {
        left: f32::INFINITY,
        top: f32::INFINITY,
        right: f32::NEG_INFINITY,
        bottom: f32::NEG_INFINITY,
    };

    pub(crate) fn add(&mut self, rect: [f32; 4]) {
        self.left = self.left.min(rect[0]);
        self.top = self.top.min(rect[1]);
        self.right = self.right.max(rect[2]);
        self.bottom = self.bottom.max(rect[3]);
    }
}

/// The scissor an unturned shared glyph run needs and the target pixels it
/// touches. A run whose quads all lie inside `scissor` needs none of its own,
/// which lets it share a draw with its neighbours. A turned run never needs
/// one: its quads were cut to its clip before the turn.
fn shared_glyph_clip(
    extent: GlyphExtent,
    scissor: TargetRect,
    viewport: ViewportUniformParams,
) -> (Option<TargetRect>, TargetRect) {
    let GlyphExtent {
        left,
        top,
        right,
        bottom,
    } = extent;
    if left > right {
        return (Some(scissor), scissor);
    }
    let left = (left - viewport.offset[0]).floor().max(0.0);
    let top = (top - viewport.offset[1]).floor().max(0.0);
    let right = (right - viewport.offset[0]).ceil();
    let bottom = (bottom - viewport.offset[1]).ceil();
    let (x, y, width, height) = scissor;
    let inside = left >= x as f32
        && top >= y as f32
        && right <= (x + width) as f32
        && bottom <= (y + height) as f32;
    if !inside || right <= left || bottom <= top {
        return (Some(scissor), scissor);
    }
    let bounds = (
        left as u32,
        top as u32,
        (right - left) as u32,
        (bottom - top) as u32,
    );
    (None, bounds)
}

fn intersect_device_rects(a: DeviceRect4, b: DeviceRect4) -> Option<DeviceRect4> {
    let left = a.0.max(b.0);
    let top = a.1.max(b.1);
    let right = (a.0 + a.2).min(b.0 + b.2);
    let bottom = (a.1 + a.3).min(b.1 + b.3);
    (right > left && bottom > top).then_some((left, top, right - left, bottom - top))
}

pub(crate) fn anchored_rect_to_device(
    rect: Rect,
    snap_anchor: Option<SnapAnchor>,
    root_scale: f32,
) -> DeviceRect4 {
    let device = anchored_device_rect(rect, snap_anchor, root_scale);
    (device.x, device.y, device.width, device.height)
}

fn mask_rect(rect: Rect) -> [f32; 4] {
    [rect.x, rect.y, rect.width, rect.height]
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct TextImageCacheKey(u64);

struct CachedTextImage {
    image: ImageBitmap,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct TextGlyphRunCacheKey(u64);

#[derive(Clone, Copy)]
struct CachedTextGlyphQuad {
    x: i32,
    y: i32,
    width: usize,
    height: usize,
    color: (f32, f32, f32, f32),
    uv: ImageUvRect,
}

struct CachedTextGlyphRun {
    /// The run's glyphs that draw: zero-sized and transparent ones are left
    /// out when the run is collected.
    glyphs: RunGlyphs,
    bounds: GlyphRunBounds,
    /// Where each of `glyphs` sits in the atlas at `atlas_generation`. A
    /// glyph's quad is derived from the two as it is drawn, so no glyph is
    /// held twice.
    atlas_entries: Option<Rc<[GlyphAtlasEntry]>>,
    atlas_generation: u64,
    /// The frame the run last drew in; see [`TEXT_GLYPH_RUN_IDLE_FRAMES`].
    last_frame: Cell<u64>,
}

/// A cached run drawn as quads: its glyphs and their atlas entries, each
/// quad derived as it is read.
#[derive(Clone, Copy)]
struct GlyphRunQuads<'a> {
    glyphs: &'a RunGlyphs,
    entries: &'a [GlyphAtlasEntry],
    /// One atlas texel's share of the atlas side: see
    /// [`TextGlyphAtlas::texel`].
    texel: f32,
    bounds: GlyphRunBounds,
}

/// The pixel box a run's drawing glyphs cover, from its raster origin.
#[derive(Clone, Copy, Debug, PartialEq)]
struct GlyphRunBounds {
    min: [i32; 2],
    max: [i32; 2],
}

impl GlyphRunBounds {
    fn of(glyphs: impl IntoIterator<Item = SoftwareGlyphAtlasPlacement>) -> Self {
        let mut bounds = Self {
            min: [i32::MAX; 2],
            max: [i32::MIN; 2],
        };
        for glyph in glyphs {
            let width = i32::try_from(glyph.width).unwrap_or(i32::MAX);
            let height = i32::try_from(glyph.height).unwrap_or(i32::MAX);
            bounds.min = [bounds.min[0].min(glyph.x), bounds.min[1].min(glyph.y)];
            bounds.max = [
                bounds.max[0].max(glyph.x.saturating_add(width)),
                bounds.max[1].max(glyph.y.saturating_add(height)),
            ];
        }
        bounds
    }

    /// The box the glyphs cover from `origin`, as left, top, right, bottom.
    fn at(self, origin: [f32; 2]) -> [f32; 4] {
        [
            origin[0] + self.min[0] as f32,
            origin[1] + self.min[1] as f32,
            origin[0] + self.max[0] as f32,
            origin[1] + self.max[1] as f32,
        ]
    }

    /// Whether every glyph of the run at `source_raster_rect` lies inside
    /// what `viewport` shows of `clip`, so none needs its own test.
    fn within_viewport(
        self,
        source_raster_rect: Rect,
        clip: Option<Rect>,
        viewport: ViewportUniformParams,
        root_scale: f32,
    ) -> bool {
        if self.min[0] > self.max[0] || !root_scale.is_finite() || root_scale <= 0.0 {
            return false;
        }
        let left = (source_raster_rect.x + self.min[0] as f32) / root_scale;
        let top = (source_raster_rect.y + self.min[1] as f32) / root_scale;
        let right = (source_raster_rect.x + self.max[0] as f32) / root_scale;
        let bottom = (source_raster_rect.y + self.max[1] as f32) / root_scale;
        let viewport_rect = viewport.scene_rect(root_scale);
        let visible = match clip {
            Some(clip) => clip.intersect(viewport_rect),
            None => Some(viewport_rect),
        };
        visible
            .is_some_and(|visible| visible.contains(left, top) && visible.contains(right, bottom))
    }
}

/// A text's glyph run as the frame found it.
struct TextGlyphRunLookup {
    glyphs: RunGlyphs,
    bounds: GlyphRunBounds,
    /// Whether the run came from the cache rather than from this frame's
    /// collection.
    cached: bool,
    /// The glyphs' atlas entries, while the atlas they were taken from holds.
    entries: Option<Rc<[GlyphAtlasEntry]>>,
}

/// A text's glyph run and where it draws.
struct TextGlyphRunDraw<'a> {
    text_draw: &'a TextDraw,
    raster_rect: Rect,
    run_key: TextGlyphRunCacheKey,
    run: TextGlyphRunLookup,
}

/// Logs why a text fell back from the glyph atlas, when
/// `CRANPOSE_TEXT_ATLAS_FALLBACK_DIAG` asks.
fn log_text_atlas_fallback(text_draw: &TextDraw) {
    if !text_atlas_fallback_diag_enabled() {
        return;
    }
    let preview: String = text_draw.text.string().text().chars().take(96).collect();
    log::warn!(
        "[text-atlas-fallback] node={:?} spans={} links={} text_len={} preview={:?} span_style={:?} paragraph_style={:?}",
        text_draw.node_id,
        text_draw.text.string().span_styles().len(),
        text_draw.text.string().links().len(),
        text_draw.text.string().text().len(),
        preview,
        text_draw.text_style.span_style,
        text_draw.text_style.paragraph_style,
    );
}

impl GlyphRunQuads<'_> {
    fn len(&self) -> usize {
        self.entries.len()
    }

    /// Each glyph as an instance at `origin`. Every glyph of a run draws: its
    /// collection leaves out the zero-sized and the transparent ones.
    fn instances_at(&self, origin: [f32; 2]) -> impl Iterator<Item = GlyphInstance> + '_ {
        let texel = self.texel;
        self.glyphs
            .boxes()
            .zip(self.entries)
            .map(move |(glyph, entry)| {
                let x0 = origin[0] + glyph.x as f32;
                let y0 = origin[1] + glyph.y as f32;
                let uv = glyph_atlas_uv_rect(*entry, texel);
                GlyphInstance {
                    rect: [
                        x0,
                        y0,
                        x0 + f32::from(glyph.width),
                        y0 + f32::from(glyph.height),
                    ],
                    uv: [uv.min[0], uv.min[1], uv.max[0], uv.max[1]],
                    uv_bounds: uv.sample_bounds,
                    color: [
                        glyph.color.0.clamp(0.0, 1.0),
                        glyph.color.1.clamp(0.0, 1.0),
                        glyph.color.2.clamp(0.0, 1.0),
                        glyph.color.3.clamp(0.0, 1.0),
                    ],
                }
            })
    }

    fn iter(&self) -> impl Iterator<Item = CachedTextGlyphQuad> + '_ {
        self.glyphs
            .boxes()
            .zip(self.entries)
            .map(|(glyph, entry)| cached_text_glyph_quad(&glyph, *entry, self.texel))
    }
}

struct CachedGpuTextGlyphRun {
    span: GlyphRunSpan,
    atlas_generation: u64,
    /// The frame the run last drew in; a run idle for
    /// [`TEXT_GLYPH_GPU_RUN_IDLE_FRAMES`] gives its quads back.
    last_frame: Cell<u64>,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
struct TextLineIndexCacheKey(usize);

struct CachedTextLineIndex {
    text: std::sync::Weak<cranpose_ui::text::RenderString>,
    len: usize,
    starts: Rc<[usize]>,
}

struct TextLineIndexCache {
    entries: BoundedLruCache<TextLineIndexCacheKey, CachedTextLineIndex>,
}

impl TextLineIndexCache {
    fn new(capacity: usize) -> Self {
        Self {
            entries: BoundedLruCache::with_capacity_at_least_one(capacity),
        }
    }

    fn line_starts(&mut self, text: &Arc<cranpose_ui::text::RenderString>) -> Rc<[usize]> {
        let key = TextLineIndexCacheKey(Arc::as_ptr(text) as usize);
        if let Some(cached) = self.entries.get(&key)
            && cached.len == text.text().len()
            && cached
                .text
                .upgrade()
                .is_some_and(|cached_text| Arc::ptr_eq(&cached_text, text))
        {
            return cached.starts.clone();
        }

        let starts = Rc::<[usize]>::from(line_start_offsets(text.text()));
        self.entries.put(
            key,
            CachedTextLineIndex {
                text: Arc::downgrade(text),
                len: text.text().len(),
                starts: starts.clone(),
            },
        );
        starts
    }
}

#[derive(Default)]
struct DeviceErrorSentry {
    errors: std::sync::atomic::AtomicU64,
    poisoned: std::sync::atomic::AtomicBool,
}

impl DeviceErrorSentry {
    fn record(&self, error: &wgpu::Error) {
        use std::sync::atomic::Ordering;
        self.poisoned.store(true, Ordering::Release);
        let count = self.errors.fetch_add(1, Ordering::Relaxed) + 1;
        if count.is_power_of_two() {
            log::error!("[gpu-device] uncaptured wgpu error #{count}: {error}");
        }
    }

    fn take_poison(&self) -> bool {
        self.poisoned
            .swap(false, std::sync::atomic::Ordering::AcqRel)
    }

    fn error_count(&self) -> u64 {
        self.errors.load(std::sync::atomic::Ordering::Relaxed)
    }
}

fn is_blend_mode_supported(mode: BlendMode) -> bool {
    matches!(
        mode,
        BlendMode::Src | BlendMode::SrcOver | BlendMode::DstOut
    )
}

pub(crate) fn blend_state_for_mode(mode: BlendMode) -> wgpu::BlendState {
    match mode {
        BlendMode::Src => wgpu::BlendState::REPLACE,
        BlendMode::DstOut => wgpu::BlendState {
            color: wgpu::BlendComponent {
                src_factor: wgpu::BlendFactor::Zero,
                dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha,
                operation: wgpu::BlendOperation::Add,
            },
            alpha: wgpu::BlendComponent {
                src_factor: wgpu::BlendFactor::Zero,
                dst_factor: wgpu::BlendFactor::OneMinusSrcAlpha,
                operation: wgpu::BlendOperation::Add,
            },
        },
        _ => wgpu::BlendState::ALPHA_BLENDING,
    }
}

pub(crate) fn supported_blend_mode(mode: BlendMode) -> BlendMode {
    if is_blend_mode_supported(mode) {
        return mode;
    }

    BlendMode::SrcOver
}

pub(crate) fn hash_f32_for_cache<H: Hasher>(value: f32, state: &mut H) {
    value.to_bits().hash(state);
}

fn hash_text_raster_geometry_for_cache<H: Hasher>(
    rect: Rect,
    static_text_motion: bool,
    state: &mut H,
) {
    hash_f32_for_cache(rect.width, state);
    hash_f32_for_cache(rect.height, state);
    static_text_motion.hash(state);
    if !static_text_motion {
        hash_f32_for_cache(rect.x.fract(), state);
        hash_f32_for_cache(rect.y.fract(), state);
    }
}

/// Hashes what a text draws apart from where and how large its rect is.
fn hash_text_draw_for_cache<H: Hasher>(
    text_draw: &TextDraw,
    raster_rect: Rect,
    text_scale: f32,
    state: &mut H,
) {
    text_draw.text.render_hash().hash(state);
    text_draw.style_hash.hash(state);
    text_draw.color.render_hash().hash(state);
    hash_text_gradient_phase_for_cache(text_draw, raster_rect, state);
    text_draw.font_size.to_bits().hash(state);
    text_scale.to_bits().hash(state);
    text_draw.layout_options.hash(state);
}

pub(crate) fn hash_text_gradient_phase_for_cache<H: Hasher>(
    text_draw: &TextDraw,
    raster_rect: Rect,
    state: &mut H,
) {
    if text_draw.text.gradient() {
        for coordinate in [raster_rect.x, raster_rect.y] {
            let phase = if coordinate >= 0.0 {
                coordinate.rem_euclid(4.0)
            } else {
                coordinate
            };
            phase.to_bits().hash(state);
        }
    }
}

/// Where a text draws in the scene, snapped, and the scale its glyphs
/// raster at; `None` for a text that draws nothing.
pub(crate) fn text_logical_geometry_for_draw(
    text_draw: &TextDraw,
    root_scale: f32,
) -> Option<(Rect, f32)> {
    if text_draw.text.is_empty()
        || text_draw.rect.width <= 0.0
        || text_draw.rect.height <= 0.0
        || !root_scale.is_finite()
        || root_scale <= 0.0
    {
        return None;
    }

    let text_scale = text_draw.scale * root_scale;
    if !text_scale.is_finite() || text_scale <= 0.0 {
        return None;
    }

    let snap_delta = text_draw
        .snap_anchor
        .map(|anchor| snap_delta_for_anchor(anchor, root_scale))
        .unwrap_or_default();
    let logical_rect = text_draw.rect.translate(snap_delta.x, snap_delta.y);
    Some((logical_rect, text_scale))
}

pub(crate) fn text_raster_geometry_for_draw(
    text_draw: &TextDraw,
    root_scale: f32,
) -> Option<(Rect, Rect, Option<Rect>, f32, bool)> {
    text_logical_geometry_for_draw(text_draw, root_scale)
        .map(|logical| text_raster_geometry_at(text_draw, logical, root_scale))
}

/// The raster geometry of a text whose logical geometry is
/// `(logical_rect, text_scale)`: see [`text_raster_geometry_for_draw`].
fn text_raster_geometry_at(
    text_draw: &TextDraw,
    (logical_rect, text_scale): (Rect, f32),
    root_scale: f32,
) -> (Rect, Rect, Option<Rect>, f32, bool) {
    let static_text_motion = text_draw.text.static_motion();
    let clip = text_draw.clip;
    let mut raster_rect = Rect {
        x: logical_rect.x * root_scale,
        y: logical_rect.y * root_scale,
        width: logical_rect.width * root_scale,
        height: logical_rect.height * root_scale,
    };
    if text_draw.snap_anchor.is_some() {
        raster_rect.x = canonicalize_device_coordinate(raster_rect.x);
        raster_rect.y = canonicalize_device_coordinate(raster_rect.y);
    }
    if static_text_motion {
        raster_rect.x = raster_rect.x.round();
        raster_rect.y = raster_rect.y.round();
    }
    raster_rect.width = raster_rect.width.ceil().max(1.0);
    raster_rect.height = raster_rect.height.ceil().max(1.0);
    (
        logical_rect,
        raster_rect,
        clip,
        text_scale,
        static_text_motion,
    )
}

fn text_draw_is_visible_in_viewport(
    logical_rect: Rect,
    clip: Option<Rect>,
    viewport: ViewportUniformParams,
    root_scale: f32,
) -> bool {
    draw_rect_is_visible_in_viewport(logical_rect, clip, viewport, root_scale)
}

fn expand_rect(rect: Rect, margin_x: f32, margin_y: f32) -> Rect {
    Rect {
        x: rect.x - margin_x,
        y: rect.y - margin_y,
        width: rect.width + margin_x * 2.0,
        height: rect.height + margin_y * 2.0,
    }
}

fn draw_rect_is_visible_in_viewport(
    rect: Rect,
    clip: Option<Rect>,
    viewport: ViewportUniformParams,
    root_scale: f32,
) -> bool {
    if !root_scale.is_finite() || root_scale <= 0.0 {
        return false;
    }
    rect_is_visible_in_rect(rect, clip, viewport.scene_rect(root_scale))
}

fn rect_is_visible_in_rect(rect: Rect, clip: Option<Rect>, viewport_rect: Rect) -> bool {
    let visible_rect = match clip {
        Some(clip) => clip.intersect(viewport_rect),
        None => Some(viewport_rect),
    };
    visible_rect.is_some_and(|visible| rect.intersect(visible).is_some())
}

fn snapped_quad_bounds(quad: [[f32; 2]; 4], anchor: Option<SnapAnchor>, root_scale: f32) -> Rect {
    let snap_delta = anchor
        .map(|anchor| snap_delta_for_anchor(anchor, root_scale))
        .unwrap_or_default();
    quad_bounds(translate_quad(quad, snap_delta))
}

/// The logical rect a draw may touch: its snapped bounds within its clip,
/// `None` when the clip leaves nothing.
fn clipped_bounds(rect: Rect, clip: Option<Rect>) -> Option<Rect> {
    match clip {
        Some(clip) => rect.intersect(clip),
        None => Some(rect),
    }
}

pub(crate) fn text_draw_bounds(text: &TextDraw, root_scale: f32) -> Option<Rect> {
    text_logical_geometry_for_draw(text, root_scale)
        .and_then(|(logical_rect, _)| clipped_bounds(logical_rect, text.clip))
}

pub(crate) fn image_draw_bounds(image: &ImageDraw, root_scale: f32) -> Option<Rect> {
    clipped_bounds(
        snapped_quad_bounds(image.quad, image.snap_anchor, root_scale),
        image.clip,
    )
}

pub(crate) fn run_draw_bounds(run: &RunDraw, root_scale: f32) -> Option<Rect> {
    let snap_delta = run
        .placement
        .snap_anchor
        .map(|anchor| snap_delta_for_anchor(anchor, root_scale))
        .unwrap_or_default();
    clipped_bounds(
        run.bounds.translate(snap_delta.x, snap_delta.y),
        run.placement.clip,
    )
}

/// The logical geometry of `text` (see [`text_logical_geometry_for_draw`])
/// when it draws any pixel inside `viewport_rect`.
pub(crate) fn text_draw_geometry_in_rect(
    text: &TextDraw,
    viewport_rect: Rect,
    root_scale: f32,
) -> Option<(Rect, f32)> {
    text_logical_geometry_for_draw(text, root_scale).filter(|(logical_rect, _)| {
        clipped_bounds(*logical_rect, text.clip)
            .is_some_and(|bounds| bounds.intersect(viewport_rect).is_some())
    })
}

fn cached_text_glyph_quad(
    glyph: &RunGlyphBox,
    entry: GlyphAtlasEntry,
    texel: f32,
) -> CachedTextGlyphQuad {
    CachedTextGlyphQuad {
        x: glyph.x,
        y: glyph.y,
        width: usize::from(glyph.width),
        height: usize::from(glyph.height),
        color: (
            glyph.color.0.clamp(0.0, 1.0),
            glyph.color.1.clamp(0.0, 1.0),
            glyph.color.2.clamp(0.0, 1.0),
            glyph.color.3.clamp(0.0, 1.0),
        ),
        uv: glyph_atlas_uv_rect(entry, texel),
    }
}

/// `quad` at `source_raster_rect`'s origin. `None` for a quad that draws
/// nothing.
fn cached_text_glyph_instance(
    source_raster_rect: Rect,
    quad: &CachedTextGlyphQuad,
) -> Option<GlyphInstance> {
    if quad.width == 0 || quad.height == 0 || quad.color.3 <= 0.0 {
        return None;
    }
    let x0 = source_raster_rect.x + quad.x as f32;
    let y0 = source_raster_rect.y + quad.y as f32;
    Some(GlyphInstance {
        rect: [x0, y0, x0 + quad.width as f32, y0 + quad.height as f32],
        uv: [
            quad.uv.min[0],
            quad.uv.min[1],
            quad.uv.max[0],
            quad.uv.max[1],
        ],
        uv_bounds: quad.uv.sample_bounds,
        color: [quad.color.0, quad.color.1, quad.color.2, quad.color.3],
    })
}

fn cached_text_glyph_quad_logical_rect(
    source_raster_rect: Rect,
    quad: &CachedTextGlyphQuad,
    root_scale: f32,
) -> Option<Rect> {
    if !root_scale.is_finite() || root_scale <= 0.0 {
        return None;
    }
    Some(Rect {
        x: (source_raster_rect.x + quad.x as f32) / root_scale,
        y: (source_raster_rect.y + quad.y as f32) / root_scale,
        width: quad.width as f32 / root_scale,
        height: quad.height as f32 / root_scale,
    })
}

fn cached_text_glyph_quad_is_visible_in_viewport(
    source_raster_rect: Rect,
    quad: &CachedTextGlyphQuad,
    clip: Option<Rect>,
    viewport: ViewportUniformParams,
    root_scale: f32,
) -> bool {
    cached_text_glyph_quad_logical_rect(source_raster_rect, quad, root_scale)
        .is_some_and(|rect| draw_rect_is_visible_in_viewport(rect, clip, viewport, root_scale))
}

pub(crate) fn hash_device_offset<H: Hasher>(
    value: f32,
    origin: f32,
    root_scale: f32,
    state: &mut H,
) {
    hash_f32_for_cache((value - origin) * root_scale, state);
}

pub(crate) fn hash_device_rect<H: Hasher>(
    rect: Rect,
    origin_x: f32,
    origin_y: f32,
    root_scale: f32,
    state: &mut H,
) {
    hash_device_offset(rect.x, origin_x, root_scale, state);
    hash_device_offset(rect.y, origin_y, root_scale, state);
    hash_device_offset(rect.width, 0.0, root_scale, state);
    hash_device_offset(rect.height, 0.0, root_scale, state);
}

/// Hashes the rect `placement` is clipped to, `clip`, and its rounding, in
/// device pixels from `origin`.
fn hash_placement_clips<H: Hasher>(
    placement: &crate::scene::Placement,
    clip: Option<Rect>,
    (origin_x, origin_y): (f32, f32),
    root_scale: f32,
    state: &mut H,
) {
    let rounding = placement.rounding;
    for (rect, radius) in [
        (clip, 0.0),
        (
            rounding.map(|rounding| rounding.rect),
            rounding.map_or(0.0, |rounding| rounding.radius),
        ),
    ] {
        rect.is_some().hash(state);
        if let Some(rect) = rect {
            let [x, y, width, height] =
                crate::run_store::placement_device_rect(placement, rect, root_scale);
            for value in [
                x - origin_x,
                y - origin_y,
                width,
                height,
                radius * root_scale,
            ] {
                hash_f32_for_cache(value, state);
            }
        }
    }
}

pub(crate) fn hash_run_item_with_clip<H: Hasher>(
    run: &RunDraw,
    clip: Option<Rect>,
    origin_x: f32,
    origin_y: f32,
    root_scale: f32,
    state: &mut H,
) {
    let tables = run.tables();
    tables.fingerprint().hash(state);
    run.segments.start.hash(state);
    run.segments.end.hash(state);
    let placement = &run.placement;
    let snap = placement
        .snap_anchor
        .map(|anchor| snap_delta_for_anchor(anchor, root_scale))
        .unwrap_or_default();
    let offset = Point::new(placement.offset.x + snap.x, placement.offset.y + snap.y);
    let canonicalize = placement.snap_anchor.is_some();
    canonicalize.hash(state);
    for segment in &tables.segments[run.segments.start as usize..run.segments.end as usize] {
        if segment.lane != RecordLane::Shapes {
            continue;
        }
        let exact_placement =
            !canonicalize || segment.kinds & ((1u8 << FRAGMENT_KIND_ARC) | LINE_KINDS) != 0;
        exact_placement.hash(state);
        if exact_placement {
            for value in [offset.x, offset.y, origin_x, origin_y, root_scale] {
                hash_f32_for_cache(value, state);
            }
            continue;
        }
        for index in segment.start..segment.start + segment.count {
            let record = tables
                .shapes
                .get(index as usize)
                .expect("shape segment record");
            let half_width = if record.is_stroked() {
                record.stroke_width * 0.5
            } else {
                0.0
            };
            let [left, top, right, bottom] = crate::run_geometry::device_shape_edges(
                record.rect,
                half_width,
                offset,
                root_scale,
                canonicalize,
            );
            for value in [left - origin_x, top - origin_y, right - left, bottom - top] {
                hash_f32_for_cache(value, state);
            }
        }
    }
    hash_placement_clips(placement, clip, (origin_x, origin_y), root_scale, state);
    if tables.segments[run.segments.start as usize..run.segments.end as usize]
        .iter()
        .any(|segment| segment.gradient || segment.vertex_gradient)
    {
        let dither = placement
            .snap_anchor
            .map(|anchor| snapped_anchor_device_origin(anchor, root_scale))
            .unwrap_or_default();
        hash_f32_for_cache(dither.x - origin_x, state);
        hash_f32_for_cache(dither.y - origin_y, state);
    }
    hash_f32_for_cache(placement.alpha, state);
    match &placement.color_filter {
        Some(filter) => {
            1u8.hash(state);
            filter.render_hash().hash(state);
        }
        None => 0u8.hash(state),
    }
}

pub(crate) fn shadow_content_hash(shadow: &ShadowDraw, root_scale: f32) -> u64 {
    let mut hasher = FxHasher::default();
    let origin = shape_shadow_bounds(shadow).unwrap_or(Rect {
        x: 0.0,
        y: 0.0,
        width: 0.0,
        height: 0.0,
    });
    let device = anchored_device_rect(
        origin,
        shadow
            .shapes
            .as_ref()
            .and_then(|run| run.placement.snap_anchor),
        root_scale,
    );
    for run in shadow.shapes.iter().chain(&shadow.post_blur_cutouts) {
        hash_run_item_with_clip(
            run,
            run.placement.clip,
            device.x.floor(),
            device.y.floor(),
            root_scale,
            &mut hasher,
        );
    }
    hasher.finish()
}

fn shape_shadow_surface_cache_key(
    shadow: &ShadowDraw,
    device_bounds: DevicePixelBounds,
    pixel_radius: f32,
    root_scale: f32,
) -> Option<ShadowSurfaceCacheKey> {
    (root_scale.is_finite() && root_scale > 0.0).then(|| ShadowSurfaceCacheKey {
        content_hash: shadow_content_hash(shadow, root_scale),
        pixel_size: [device_bounds.width, device_bounds.height],
        root_scale_bits: root_scale.to_bits(),
        blur_radius_bits: pixel_radius.to_bits(),
    })
}

fn shape_shadow_bounds(shadow: &ShadowDraw) -> Option<Rect> {
    shadow.shapes.as_ref().map(|run| run.bounds)
}

pub(crate) fn shadow_draw_bounds(shadow: &ShadowDraw) -> Option<Rect> {
    shape_shadow_bounds(shadow)
        .into_iter()
        .chain(shadow.texts.iter().map(|text| text.rect))
        .reduce(|a, b| Rect {
            x: a.x.min(b.x),
            y: a.y.min(b.y),
            width: (a.x + a.width).max(b.x + b.width) - a.x.min(b.x),
            height: (a.y + a.height).max(b.y + b.height) - a.y.min(b.y),
        })
}

/// The shape shader's source for `mode`, built when its module is parsed.
fn shape_shader_code(mode: RunBufferMode) -> ShaderCode {
    if mode.trig_fill {
        ShaderCode::wgsl(|| Cow::Owned(shaders::filling_shape_shader()))
    } else if mode.storage {
        ShaderCode::wgsl(|| Cow::Owned(shaders::storage_shape_shader()))
    } else {
        shaders::SHAPE
    }
}

/// The vertex entry point of every fullscreen strip.
pub(crate) const FULLSCREEN_VS: &str = "fullscreen_vs";

/// A pipeline that draws one full-screen triangle strip from `fullscreen_vs`
/// into a single color target, the shape every effect and composite pass
/// shares; `constants` fixes the shader's override constants.
#[expect(clippy::too_many_arguments)]
pub(crate) fn create_fullscreen_strip_pipeline(
    device: &wgpu::Device,
    cache: Option<&wgpu::PipelineCache>,
    log_label: &str,
    label: &'static str,
    layout: &wgpu::PipelineLayout,
    (vertex_module, fragment_module): (&wgpu::ShaderModule, &wgpu::ShaderModule),
    (vertex_entry, fragment_entry): (&'static str, &'static str),
    constants: &[(&str, f64)],
    (target, depth_stencil): (wgpu::ColorTargetState, Option<wgpu::DepthStencilState>),
) -> wgpu::RenderPipeline {
    create_render_pipeline_logged(
        device,
        cache,
        log_label,
        wgpu::RenderPipelineDescriptor {
            label: Some(label),
            layout: Some(layout),
            vertex: wgpu::VertexState {
                module: vertex_module,
                entry_point: Some(vertex_entry),
                buffers: &[],
                compilation_options: wgpu::PipelineCompilationOptions {
                    constants,
                    ..wgpu::PipelineCompilationOptions::default()
                },
            },
            fragment: Some(wgpu::FragmentState {
                module: fragment_module,
                entry_point: Some(fragment_entry),
                targets: &[Some(target)],
                compilation_options: wgpu::PipelineCompilationOptions {
                    constants,
                    ..wgpu::PipelineCompilationOptions::default()
                },
            }),
            primitive: wgpu::PrimitiveState {
                topology: wgpu::PrimitiveTopology::TriangleStrip,
                strip_index_format: None,
                front_face: wgpu::FrontFace::Ccw,
                cull_mode: None,
                ..Default::default()
            },
            depth_stencil,
            multisample: wgpu::MultisampleState::default(),
            multiview_mask: None,
            cache: None,
        },
    )
}

pub(crate) fn create_render_pipeline_logged<'a>(
    device: &wgpu::Device,
    cache: Option<&'a wgpu::PipelineCache>,
    tag: &str,
    mut descriptor: wgpu::RenderPipelineDescriptor<'a>,
) -> wgpu::RenderPipeline {
    descriptor.cache = cache;
    build_pipeline_logged(tag, || device.create_render_pipeline(&descriptor))
}

pub(crate) fn build_pipeline_logged<T>(tag: &str, build: impl FnOnce() -> T) -> T {
    let started = Instant::now();
    let pipeline = build();
    log::info!(
        "[pipeline-create] {tag} {:.1}ms on {}",
        instant_ms(started, Instant::now()),
        std::thread::current().name().unwrap_or("unnamed thread"),
    );
    crate::thread_state::with(|state| {
        if state.off_frame_builds.get() {
            PIPELINES_CREATED_OFF_FRAME.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
        } else {
            PIPELINES_CREATED.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
            state.thread_builds.set(state.thread_builds.get() + 1);
        }
    });
    #[cfg(not(target_arch = "wasm32"))]
    crate::pipeline_disk_cache::note_change();
    pipeline
}

/// Pipelines this process has built on a thread that draws.
///
/// A build runs the backend's shader compiler, and whoever asks for one while
/// drawing waits for it there. A count that grows across an interaction names
/// work a person waited on, whatever the driver's own caches made a compile
/// cost on this machine. Builds handed to [`crate::pipeline_compiler`] are
/// counted apart, by [`pipelines_created_off_frame`]: they cost a frame
/// nothing.
static PIPELINES_CREATED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static PIPELINES_CREATED_OFF_FRAME: std::sync::atomic::AtomicU64 =
    std::sync::atomic::AtomicU64::new(0);

/// The pipelines this thread has built while drawing: a frame that drew
/// with this unchanged built nothing itself.
#[cfg(not(target_arch = "wasm32"))]
fn thread_builds() -> u64 {
    crate::thread_state::with(|state| state.thread_builds.get())
}

/// Declares that pipelines built on this thread are built away from any
/// frame. The compiler thread says so once, when it starts.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn mark_thread_off_frame() {
    crate::thread_state::with(|state| state.off_frame_builds.set(true));
}

pub fn pipelines_created() -> u64 {
    PIPELINES_CREATED.load(std::sync::atomic::Ordering::Relaxed)
}

/// Pipelines built away from every frame, on the compiler thread.
pub fn pipelines_created_off_frame() -> u64 {
    PIPELINES_CREATED_OFF_FRAME.load(std::sync::atomic::Ordering::Relaxed)
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[repr(u8)]
pub(crate) enum RunTier {
    Store = 0,
    Arena = 1,
    Either = 2,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub(crate) struct ShapeVariant {
    kinds: u8,
    brush: Option<u8>,
    solid: bool,
    /// How its records meet their placements' clips. A rounded clip gives
    /// every fragment its coverage, on the general entry points.
    clip: SegmentClip,
    interior: bool,
    dither: bool,
    ablation: ShapeAblation,
}

const FLAT_FILL_ENTRIES: [[(&str, &str); 2]; 2] = [
    [
        ("vs_record_plain_fill", "fs_plain_fill"),
        ("vs_record_dithered_fill", "fs_dithered_fill"),
    ],
    [
        ("vs_record_clipped_fill", "fs_clipped_fill"),
        ("vs_record_solid", "fs_solid"),
    ],
];

/// The entry points of a variant that draws slices of path fills alone and
/// flat: a solid brush, and a gradient between two stops from the vertices.
const SLICE_ENTRIES: [(&str, &str); 2] = [
    ("vs_record_slice", "fs_slice"),
    ("vs_record_gradient_slice", "fs_gradient_slice"),
];

const ALL_SHAPE_KINDS: u8 =
    (1 << FRAGMENT_KIND_FILL) | (1 << FRAGMENT_KIND_STROKE) | (1 << FRAGMENT_KIND_ARC) | LINE_KINDS;
/// The kinds the shader draws from line records: segments, and slices of
/// path fills. A variant compiles in the code of only those it draws.
const LINE_KINDS: u8 = (1 << FRAGMENT_KIND_LINE) | TRAPEZOID_KIND;
const TRAPEZOID_KIND: u8 = 1 << FRAGMENT_KIND_TRAPEZOID;

fn variant_kinds(kinds: u8) -> u8 {
    if kinds.count_ones() <= 1 {
        kinds
    } else {
        (ALL_SHAPE_KINDS & !LINE_KINDS) | (kinds & LINE_KINDS)
    }
}

impl ShapeVariant {
    /// Every kind but slices of path fills, which only a segment holding
    /// them compiles in.
    const GENERAL: Self = Self {
        kinds: ALL_SHAPE_KINDS & !TRAPEZOID_KIND,
        brush: None,
        solid: false,
        clip: SegmentClip::Tested,
        interior: true,
        dither: false,
        ablation: ShapeAblation {
            material: false,
            fill: false,
        },
    };

    /// The variant `segment` draws with. `vertex_gradients` says a rect
    /// fill may take its gradient from its vertices here; a segment of
    /// slices of path fills may wherever it is drawn `flat`. Under a rounded
    /// clip the variant keeps the general entry points, which carry the
    /// clip's radius to the fragment, but compiles in only the segment's
    /// kinds and brush: the general program with every kind and brush is
    /// twice the code (Mali-G76: 108-115 ms to compile or link, the
    /// specialized ones 22-62).
    pub(crate) fn of_segment(
        segment: &RecordSegment,
        clip: SegmentClip,
        ablation: ShapeAblation,
        laid: bool,
        (flat, vertex_gradients): (bool, bool),
    ) -> Self {
        if !shape_variants_enabled() {
            return Self {
                kinds: Self::GENERAL.kinds | (segment.kinds & TRAPEZOID_KIND),
                ablation,
                ..Self::GENERAL
            }
            .clipped_by(clip);
        }
        let vertex_gradients = if segment.kinds == TRAPEZOID_KIND {
            flat
        } else {
            vertex_gradients
        };
        let gradient = segment.gradient || (segment.vertex_gradient && !vertex_gradients);
        Self {
            kinds: variant_kinds(segment.kinds),
            brush: gradient
                .then(|| segment.uniform_brush())
                .flatten()
                .map(|brush| brush as u8),
            solid: !gradient,
            clip,
            // A rounded clip's corners are not opaque: no interior path.
            interior: clip != SegmentClip::Rounded
                && gradient
                && segment.interiors
                && !(laid && segment.occluders && !segment.bare_interiors),
            dither: segment.vertex_gradient && vertex_gradients,
            ablation,
        }
    }

    /// The one kind the variant draws, as the shader's packed kind names
    /// it: a slice of a path fill is a line record.
    fn kind(self) -> Option<u8> {
        (self.kinds.count_ones() == 1).then(|| match self.kinds.trailing_zeros() {
            FRAGMENT_KIND_TRAPEZOID => FRAGMENT_KIND_LINE as u8,
            kind => kind as u8,
        })
    }

    /// The vertex and fragment entry points of this variant, for records
    /// drawn `flat` (none of them turned).
    fn entries(self, flat: bool) -> (&'static str, &'static str) {
        let fill = self.kind() == Some(FRAGMENT_KIND_FILL as u8);
        if self.rounded() {
            ("vs_record", "fs_main")
        } else if self.solid && flat && self.kinds == TRAPEZOID_KIND {
            SLICE_ENTRIES[usize::from(self.dither)]
        } else if self.solid && fill && flat {
            FLAT_FILL_ENTRIES[usize::from(self.clip == SegmentClip::Tested)]
                [usize::from(self.dither)]
        } else if self.solid {
            ("vs_record_solid", "fs_solid")
        } else if fill && !self.ablation.material {
            ("vs_record_gradient_fill", "fs_gradient_fill")
        } else {
            ("vs_record", "fs_main")
        }
    }

    fn general(self) -> Self {
        Self {
            kinds: (ALL_SHAPE_KINDS & !LINE_KINDS) | (self.kinds & LINE_KINDS),
            ablation: self.ablation,
            ..Self::GENERAL
        }
        .clipped_by(self.clip)
    }

    /// The variant, keeping the rounded clip `clip` names: the general
    /// variant tests every rect clip, but no fallback may drop a rounding.
    fn clipped_by(self, clip: SegmentClip) -> Self {
        if clip == SegmentClip::Rounded {
            Self { clip, ..self }
        } else {
            self
        }
    }

    fn rounded(self) -> bool {
        self.clip == SegmentClip::Rounded
    }

    fn joined(self) -> Self {
        if self.solid {
            Self {
                dither: true,
                ..self
            }
        } else {
            Self {
                interior: true,
                ..self
            }
        }
    }
}

/// How a segment's draw meets its placement's clip: a scissor or no clip
/// leaves the shader nothing to test, a rect clip off the scissor's grid is
/// tested per fragment, and a rounded clip gives every fragment coverage.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub(crate) enum SegmentClip {
    Untested,
    Tested,
    Rounded,
}

impl SegmentClip {
    /// The clip a draw under `placement` tests, given whether its rect clip
    /// became the draw's scissor.
    pub(crate) fn of(placement: &crate::scene::Placement, scissored: bool) -> Self {
        if placement.clip_rounded() {
            Self::Rounded
        } else if placement.clip.is_some() && !scissored {
            Self::Tested
        } else {
            Self::Untested
        }
    }
}

static SHAPE_VARIANTS: DebugToggle = DebugToggle::new("CRANPOSE_SHAPE_VARIANTS");

fn shape_variants_enabled() -> bool {
    !SHAPE_VARIANTS.equals("0")
}

/// What every draw of one pass shares: its target's size and whether the
/// pass has a depth buffer its opaque interiors fill first.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct PassFrame {
    pub(crate) size: (u32, u32),
    pub(crate) depth: bool,
}

impl PassFrame {
    /// The scissor a draw bounded by `scissor`, or by nothing, sets.
    pub(crate) fn scissor(self, scissor: Option<(u32, u32, u32, u32)>) -> (u32, u32, u32, u32) {
        scissor.unwrap_or((0, 0, self.size.0, self.size.1))
    }
}

/// Which of a pass's two stages a shape draw records: the front-to-back
/// opaque interiors, or the paint in draw order.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum RunStage {
    Interiors,
    Paint,
}

/// The depth buffer of a pass that lays opaque interiors down first.
pub(crate) const DEPTH_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float;

/// How a pipeline meets its pass's depth buffer.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub(crate) enum ShapeDepth {
    /// The pass has no depth buffer.
    Off,
    /// Paints what the opaque interiors of later records leave visible.
    Tested,
    /// Lays down opaque interiors, front to back, ahead of the paint.
    Interior,
}

impl ShapeDepth {
    fn stencil_state(self) -> Option<wgpu::DepthStencilState> {
        let (depth_write_enabled, depth_compare) = match self {
            Self::Off => return None,
            Self::Tested => (false, wgpu::CompareFunction::Less),
            Self::Interior => (true, wgpu::CompareFunction::Less),
        };
        Some(wgpu::DepthStencilState {
            format: DEPTH_FORMAT,
            depth_write_enabled: Some(depth_write_enabled),
            depth_compare: Some(depth_compare),
            stencil: wgpu::StencilState::default(),
            bias: wgpu::DepthBiasState::default(),
        })
    }
}

/// Depth steps one draw apart in a pass's order, as the shaders'
/// `DEPTH_STEP`.
const PASS_DEPTH_STEP: f32 = 1.0 / 1_048_576.0;

/// The depth of the draw at `index` in its pass's order, as the shaders'
/// `record_depth` and `batch_depth` place it: later draws nearer.
pub(crate) fn pass_order_depth(index: u32) -> f32 {
    (1.0 - (index as f32 + 1.0) * PASS_DEPTH_STEP).max(0.0)
}

/// The depth state of a glyph or image pipeline: in a pass with a depth
/// buffer it paints what later opaque interiors leave visible, as shapes do.
pub(crate) fn overlay_depth_state(depth: bool) -> Option<wgpu::DepthStencilState> {
    if depth {
        ShapeDepth::Tested.stencil_state()
    } else {
        None
    }
}

/// Which records a shape pipeline draws turned, as `SHAPE_TURNS` in the
/// shape shader: none, every one, or each as its placement says.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub(crate) enum ShapeTurns {
    None,
    All,
    Mixed,
}

impl ShapeTurns {
    /// The turns of records placed under `turn`, in a pass whose flat and
    /// turned records alternate often when `mixed`.
    pub(crate) fn of(turn: SegmentTransform, mixed: bool) -> Self {
        if mixed {
            Self::Mixed
        } else if turn.is_identity() {
            Self::None
        } else {
            Self::All
        }
    }

    fn constant(self) -> f64 {
        match self {
            Self::None => 0.0,
            Self::All => 1.0,
            Self::Mixed => 2.0,
        }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub(crate) struct ShapePipelineKey {
    pub(crate) blend_mode: BlendMode,
    pub(crate) tier: RunTier,
    pub(crate) variant: ShapeVariant,
    pub(crate) turns: ShapeTurns,
    pub(crate) depth: ShapeDepth,
}

/// Packs a key's fields into a number, low bit first.
#[derive(Default)]
struct KeyWriter {
    bits: u64,
    width: u32,
}

impl KeyWriter {
    fn put(&mut self, value: u64, width: u32) {
        self.bits |= value << self.width;
        self.width += width;
    }

    fn put_byte(&mut self, value: Option<u8>) {
        self.put(u64::from(value.is_some()), 1);
        self.put(u64::from(value.unwrap_or_default()), 8);
    }
}

/// Unpacks what [`KeyWriter`] packed, low bit first.
/// The bits a key's shape kinds take: one per fragment kind.
const KIND_BITS: u32 = FRAGMENT_KIND_TRAPEZOID + 1;

struct KeyReader(u64);

impl KeyReader {
    fn take(&mut self, width: u32) -> u64 {
        let value = self.0 & ((1 << width) - 1);
        self.0 >>= width;
        value
    }

    fn take_flag(&mut self) -> bool {
        self.take(1) == 1
    }

    fn take_byte(&mut self) -> Option<u8> {
        let present = self.take_flag();
        let value = u8::try_from(self.take(8)).ok()?;
        present.then_some(value)
    }

    fn take_kinds(&mut self) -> Option<u8> {
        u8::try_from(self.take(KIND_BITS))
            .ok()
            .filter(|kinds| *kinds != 0)
    }
}

impl ShapePipelineKey {
    pub(crate) const DISK_LAYOUT: [u8; 8] = *b"CPKEY003";

    /// The flat solid rect fill of a pass without a depth buffer, which
    /// every screen draws: on all 38 demo pages, and in a launch's first
    /// frames, which draw without the depth pre-pass.
    #[cfg(not(target_arch = "wasm32"))]
    const SOLID_RECT: Self = Self {
        blend_mode: BlendMode::SrcOver,
        tier: RunTier::Arena,
        variant: ShapeVariant {
            kinds: 1 << FRAGMENT_KIND_FILL,
            brush: None,
            solid: true,
            clip: SegmentClip::Untested,
            interior: false,
            dither: false,
            ablation: ShapeAblation {
                material: false,
                fill: false,
            },
        },
        turns: ShapeTurns::None,
        depth: ShapeDepth::Off,
    };

    /// The key as a number a later launch reads back with
    /// [`Self::from_bits`], to build the pipeline ahead of its first frame.
    pub(crate) fn to_bits(self) -> u64 {
        let mut bits = KeyWriter::default();
        bits.put(self.blend_mode as u64, 5);
        bits.put(self.tier as u64, 2);
        let variant = self.variant;
        bits.put(u64::from(variant.kinds), KIND_BITS);
        bits.put_byte(variant.brush);
        bits.put(u64::from(variant.solid), 1);
        bits.put(
            match variant.clip {
                SegmentClip::Untested => 0,
                SegmentClip::Tested => 1,
                SegmentClip::Rounded => 2,
            },
            2,
        );
        bits.put(u64::from(variant.interior), 1);
        bits.put(u64::from(variant.dither), 1);
        bits.put(u64::from(variant.ablation.material), 1);
        bits.put(u64::from(variant.ablation.fill), 1);
        bits.put(
            match self.turns {
                ShapeTurns::None => 0,
                ShapeTurns::All => 1,
                ShapeTurns::Mixed => 2,
            },
            2,
        );
        bits.put(
            match self.depth {
                ShapeDepth::Off => 0,
                ShapeDepth::Tested => 1,
                ShapeDepth::Interior => 2,
            },
            2,
        );
        bits.bits
    }

    /// The key [`Self::to_bits`] packed, or `None` for a number no key
    /// packs to.
    pub(crate) fn from_bits(packed: u64) -> Option<Self> {
        let mut bits = KeyReader(packed);
        let blend_mode = *BlendMode::ALL.get(usize::try_from(bits.take(5)).ok()?)?;
        let tier = match bits.take(2) {
            0 => RunTier::Store,
            1 => RunTier::Arena,
            2 => RunTier::Either,
            _ => return None,
        };
        let kinds = bits.take_kinds()?;
        let brush = bits.take_byte();
        let solid = bits.take_flag();
        let clip = match bits.take(2) {
            0 => SegmentClip::Untested,
            1 => SegmentClip::Tested,
            2 => SegmentClip::Rounded,
            _ => return None,
        };
        let interior = bits.take_flag();
        let dither = bits.take_flag();
        let ablation = ShapeAblation {
            material: bits.take_flag(),
            fill: bits.take_flag(),
        };
        let turns = match bits.take(2) {
            0 => ShapeTurns::None,
            1 => ShapeTurns::All,
            2 => ShapeTurns::Mixed,
            _ => return None,
        };
        let depth = match bits.take(2) {
            0 => ShapeDepth::Off,
            1 => ShapeDepth::Tested,
            2 => ShapeDepth::Interior,
            _ => return None,
        };
        (bits.0 == 0).then_some(Self {
            blend_mode,
            tier,
            variant: ShapeVariant {
                kinds,
                brush,
                solid,
                clip,
                interior,
                dither,
                ablation,
            },
            turns,
            depth,
        })
    }

    #[cfg(test)]
    pub(crate) fn general_for(blend_mode: BlendMode, tier: RunTier) -> Self {
        Self {
            blend_mode,
            tier,
            variant: ShapeVariant::GENERAL,
            turns: ShapeTurns::None,
            depth: ShapeDepth::Off,
        }
    }

    /// The pipeline that lays down the opaque interiors of this key's
    /// draws, if they have any: plain source-over draws. A turned record's
    /// layer draws in place only under a rigid turn, which keeps the
    /// interior's half-pixel inset from the fill's edge exact. The interior
    /// shader reads only the tier and the turns, so every variant shares one
    /// pipeline per tier and turns: one compile, and draws of different
    /// variants lay their interiors down together.
    pub(crate) fn interior(self) -> Option<Self> {
        (self.depth == ShapeDepth::Tested
            && !self.variant.rounded()
            && self.blend_mode == BlendMode::SrcOver
            && !self.variant.ablation.material
            && !self.variant.ablation.fill)
            .then_some(Self {
                variant: ShapeVariant::GENERAL,
                depth: ShapeDepth::Interior,
                ..self
            })
    }

    pub(crate) fn general(self) -> Self {
        Self {
            tier: RunTier::Either,
            variant: self.variant.general(),
            ..self
        }
    }

    pub(crate) fn is_general(self) -> bool {
        self.variant == self.variant.general()
    }

    pub(crate) fn joined(self) -> Self {
        Self {
            variant: self.variant.joined(),
            ..self
        }
    }
}

/// The entry points and pipeline constants of the shape program a key names.
pub(crate) struct ShapeProgram {
    pub(crate) vertex_entry: &'static str,
    pub(crate) fragment_entry: &'static str,
    pub(crate) constants: [(&'static str, f64); 14],
}

impl ShapePipelineKey {
    pub(crate) fn program(self, mode: RunBufferMode) -> ShapeProgram {
        let Self {
            tier,
            variant,
            turns,
            depth,
            ..
        } = self;
        let constants = [
            ("SHAPE_KIND_FIXED", variant.kind().map_or(-1.0, f64::from)),
            ("SHAPE_KINDS", f64::from(variant.kinds)),
            ("BRUSH_KIND_FIXED", variant.brush.map_or(-1.0, f64::from)),
            ("SHAPE_SOLID", f64::from(u8::from(variant.solid))),
            (
                "SHAPE_CLIPPED",
                f64::from(u8::from(variant.clip != SegmentClip::Untested)),
            ),
            ("SHAPE_ROUNDED_CLIP", f64::from(u8::from(variant.rounded()))),
            (
                "SHAPE_INTERIOR",
                f64::from(u8::from(variant.interior && !variant.rounded())),
            ),
            ("SHAPE_DITHER", f64::from(u8::from(variant.dither))),
            ("SHAPE_TURNS", turns.constant()),
            ("SHAPE_TIER", f64::from(tier as u8)),
            ("SHAPE_BANDS", f64::from(u8::from(mode.storage))),
            ("SHAPE_TRIG_FILLED", f64::from(u8::from(mode.trig_fill))),
            ("SHAPE_FLAT", f64::from(u8::from(variant.ablation.material))),
            ("SHAPE_DISCARD", f64::from(u8::from(variant.ablation.fill))),
        ];
        let (vertex_entry, fragment_entry) = if depth == ShapeDepth::Interior {
            ("vs_record_interior", "fs_interior")
        } else {
            variant.entries(turns == ShapeTurns::None)
        };
        ShapeProgram {
            vertex_entry,
            fragment_entry,
            constants,
        }
    }
}

pub(crate) fn create_shape_pipeline(
    device: &wgpu::Device,
    cache: Option<&wgpu::PipelineCache>,
    surface_format: wgpu::TextureFormat,
    shader: &SharedShader,
    key: ShapePipelineKey,
    mode: RunBufferMode,
) -> wgpu::RenderPipeline {
    let ShapePipelineKey {
        blend_mode,
        tier,
        variant,
        turns,
        depth,
    } = key;
    let ShapeProgram {
        vertex_entry,
        fragment_entry,
        constants,
    } = key.program(mode);
    let blend = (depth != ShapeDepth::Interior).then(|| blend_state_for_mode(blend_mode));
    let instance_layout = record_vertex_layouts().map(Some);
    let stages = shader.stages((vertex_entry, fragment_entry), &constants);

    create_render_pipeline_logged(
        device,
        cache,
        &format!(
            "shape blend={blend_mode:?} tier={tier:?} variant={variant:?} turns={turns:?} depth={depth:?}"
        ),
        wgpu::RenderPipelineDescriptor {
            label: Some("Shape Pipeline"),
            layout: Some(shader.layout()),
            vertex: wgpu::VertexState {
                module: &stages.vertex,
                entry_point: Some(vertex_entry),
                compilation_options: wgpu::PipelineCompilationOptions {
                    constants: stages.constants,
                    ..wgpu::PipelineCompilationOptions::default()
                },
                buffers: &instance_layout,
            },
            fragment: Some(wgpu::FragmentState {
                module: &stages.fragment,
                entry_point: Some(fragment_entry),
                compilation_options: wgpu::PipelineCompilationOptions {
                    constants: stages.constants,
                    ..wgpu::PipelineCompilationOptions::default()
                },
                targets: &[Some(wgpu::ColorTargetState {
                    format: surface_format,
                    blend,
                    write_mask: wgpu::ColorWrites::ALL,
                })],
            }),
            primitive: wgpu::PrimitiveState {
                topology: wgpu::PrimitiveTopology::TriangleList,
                strip_index_format: None,
                front_face: wgpu::FrontFace::Ccw,
                cull_mode: None,
                unclipped_depth: false,
                polygon_mode: wgpu::PolygonMode::Fill,
                conservative: false,
            },
            depth_stencil: depth.stencil_state(),
            multisample: wgpu::MultisampleState::default(),
            multiview_mask: None,
            cache: None,
        },
    )
}
fn create_image_pipeline(
    device: &wgpu::Device,
    cache: Option<&wgpu::PipelineCache>,
    surface_format: wgpu::TextureFormat,
    shader: &SharedShader,
    blend_mode: BlendMode,
    variant: ImageVariant,
) -> wgpu::RenderPipeline {
    let module = shader.module();
    let ImageVariant {
        depth,
        alpha_mask,
        rounded,
    } = variant;
    let (vertex_entry, fragment_entry) = match (alpha_mask, rounded) {
        (false, false) => ("image_vs_main", "image_fs_main"),
        (true, false) => ("image_vs_main", "image_mask_fs_main"),
        (false, true) => ("image_rounded_vs_main", "image_rounded_fs_main"),
        (true, true) => ("image_rounded_vs_main", "image_mask_rounded_fs_main"),
    };
    // A rounded image reads its clip from an instance beside its vertices.
    let buffers = [Some(Vertex::desc()), Some(DeviceRoundedClip::desc())];
    create_render_pipeline_logged(
        device,
        cache,
        &format!(
            "image blend={blend_mode:?} depth={depth} alpha_mask={alpha_mask} rounded={rounded}"
        ),
        wgpu::RenderPipelineDescriptor {
            label: Some("Image Pipeline"),
            layout: Some(shader.layout()),
            vertex: wgpu::VertexState {
                module,
                entry_point: Some(vertex_entry),
                compilation_options: wgpu::PipelineCompilationOptions::default(),
                buffers: if rounded { &buffers } else { &buffers[..1] },
            },
            fragment: Some(wgpu::FragmentState {
                module,
                entry_point: Some(fragment_entry),
                compilation_options: wgpu::PipelineCompilationOptions::default(),
                targets: &[Some(wgpu::ColorTargetState {
                    format: surface_format,
                    blend: Some(blend_state_for_mode(blend_mode)),
                    write_mask: wgpu::ColorWrites::ALL,
                })],
            }),
            primitive: wgpu::PrimitiveState {
                topology: wgpu::PrimitiveTopology::TriangleList,
                strip_index_format: None,
                front_face: wgpu::FrontFace::Ccw,
                cull_mode: None,
                unclipped_depth: false,
                polygon_mode: wgpu::PolygonMode::Fill,
                conservative: false,
            },
            depth_stencil: overlay_depth_state(depth),
            multisample: wgpu::MultisampleState::default(),
            multiview_mask: None,
            cache: None,
        },
    )
}

fn create_glyph_atlas_pipeline(
    device: &wgpu::Device,
    cache: Option<&wgpu::PipelineCache>,
    surface_format: wgpu::TextureFormat,
    shader: &SharedShader,
    (depth, quads): (bool, GlyphQuads),
) -> wgpu::RenderPipeline {
    let module = shader.module();
    create_render_pipeline_logged(
        device,
        cache,
        match (depth, quads) {
            (false, GlyphQuads::Plain) => "glyph-atlas",
            (true, GlyphQuads::Plain) => "glyph-atlas depth",
            (false, GlyphQuads::Aligned) => "glyph-atlas aligned",
            (true, GlyphQuads::Aligned) => "glyph-atlas aligned depth",
            (false, GlyphQuads::Turned) => "glyph-atlas turned",
            (true, GlyphQuads::Turned) => "glyph-atlas turned depth",
        },
        wgpu::RenderPipelineDescriptor {
            label: Some("Glyph Atlas Pipeline"),
            layout: Some(shader.layout()),
            vertex: wgpu::VertexState {
                module,
                entry_point: Some(match quads {
                    GlyphQuads::Plain => "glyph_atlas_vs_main",
                    GlyphQuads::Aligned => "glyph_atlas_aligned_vs_main",
                    GlyphQuads::Turned => "glyph_atlas_turned_vs_main",
                }),
                compilation_options: wgpu::PipelineCompilationOptions::default(),
                buffers: &[Some(match quads {
                    GlyphQuads::Turned => TurnedGlyph::desc(),
                    GlyphQuads::Plain | GlyphQuads::Aligned => GlyphInstance::desc(),
                })],
            },
            fragment: Some(wgpu::FragmentState {
                module,
                entry_point: Some(match quads {
                    GlyphQuads::Aligned => "glyph_atlas_aligned_fs_main",
                    GlyphQuads::Plain | GlyphQuads::Turned => "glyph_atlas_fs_main",
                }),
                compilation_options: wgpu::PipelineCompilationOptions::default(),
                targets: &[Some(wgpu::ColorTargetState {
                    format: surface_format,
                    blend: Some(blend_state_for_mode(BlendMode::SrcOver)),
                    write_mask: wgpu::ColorWrites::ALL,
                })],
            }),
            primitive: wgpu::PrimitiveState {
                topology: wgpu::PrimitiveTopology::TriangleStrip,
                strip_index_format: None,
                front_face: wgpu::FrontFace::Ccw,
                cull_mode: None,
                unclipped_depth: false,
                polygon_mode: wgpu::PolygonMode::Fill,
                conservative: false,
            },
            depth_stencil: overlay_depth_state(depth),
            multisample: wgpu::MultisampleState::default(),
            multiview_mask: None,
            cache: None,
        },
    )
}

#[repr(C)]
#[derive(Copy, Clone, Debug, Pod, Zeroable)]
pub(crate) struct Vertex {
    position: [f32; 2],
    color: [f32; 4],
    uv: [f32; 2],
    uv_bounds: [f32; 4],
}

impl Vertex {
    const ATTRIBS: [wgpu::VertexAttribute; 4] = wgpu::vertex_attr_array![
        0 => Float32x2,
        1 => Float32x4,
        2 => Float32x2,
        3 => Float32x4,
    ];

    fn desc() -> wgpu::VertexBufferLayout<'static> {
        wgpu::VertexBufferLayout {
            array_stride: std::mem::size_of::<Vertex>() as wgpu::BufferAddress,
            step_mode: wgpu::VertexStepMode::Vertex,
            attributes: &Self::ATTRIBS,
        }
    }
}

/// The corners a glyph instance draws as a triangle strip: top-left,
/// top-right, bottom-left, bottom-right.
const GLYPH_QUAD_CORNERS: u32 = 4;

/// One glyph quad as the glyph pipeline draws it: an instance whose four
/// corners the vertex stage picks from `rect` and `uv`.
#[repr(C)]
#[derive(Copy, Clone, Debug, PartialEq, Pod, Zeroable)]
pub(crate) struct GlyphInstance {
    rect: [f32; 4],
    uv: [f32; 4],
    uv_bounds: [f32; 4],
    color: [f32; 4],
}

impl GlyphInstance {
    const ATTRIBS: [wgpu::VertexAttribute; 4] = wgpu::vertex_attr_array![
        0 => Float32x4,
        1 => Float32x4,
        2 => Float32x4,
        3 => Float32x4
    ];

    fn desc() -> wgpu::VertexBufferLayout<'static> {
        wgpu::VertexBufferLayout {
            array_stride: std::mem::size_of::<GlyphInstance>() as wgpu::BufferAddress,
            step_mode: wgpu::VertexStepMode::Instance,
            attributes: &Self::ATTRIBS,
        }
    }

    /// The part of the quad inside `edges` (left, top, right, bottom, in the
    /// quad's own space), its atlas coordinates cut to match; `None` when
    /// none of it is inside.
    pub(crate) fn clipped_to(self, edges: [f32; 4]) -> Option<Self> {
        let [x0, y0, x1, y1] = self.rect;
        let rect = [
            x0.max(edges[0]),
            y0.max(edges[1]),
            x1.min(edges[2]),
            y1.min(edges[3]),
        ];
        if rect[2] <= rect[0] || rect[3] <= rect[1] {
            return None;
        }
        if rect == self.rect {
            return Some(self);
        }
        let [u0, v0, u1, v1] = self.uv;
        let u = |x: f32| u0 + (x - x0) / (x1 - x0) * (u1 - u0);
        let v = |y: f32| v0 + (y - y0) / (y1 - y0) * (v1 - v0);
        Some(Self {
            rect,
            uv: [u(rect[0]), v(rect[1]), u(rect[2]), v(rect[3])],
            ..self
        })
    }
}

/// Hands `sink` each quad of a glyph run at `source_raster_rect` that the
/// viewport shows, as an instance cut to `cut` when the text is cut.
#[inline]
fn for_each_visible_glyph(
    source_raster_rect: Rect,
    quads: GlyphRunQuads<'_>,
    (clip, cut): (Option<Rect>, Option<[f32; 4]>),
    viewport: ViewportUniformParams,
    root_scale: f32,
    mut sink: impl FnMut(GlyphInstance),
) {
    let whole_run_visible =
        quads
            .bounds
            .within_viewport(source_raster_rect, clip, viewport, root_scale);
    for quad in quads.iter() {
        if !whole_run_visible
            && !cached_text_glyph_quad_is_visible_in_viewport(
                source_raster_rect,
                &quad,
                clip,
                viewport,
                root_scale,
            )
        {
            continue;
        }
        let Some(glyph) = cached_text_glyph_instance(source_raster_rect, &quad) else {
            continue;
        };
        let glyph = match cut {
            Some(edges) => glyph.clipped_to(edges),
            None => Some(glyph),
        };
        if let Some(glyph) = glyph {
            sink(glyph);
        }
    }
}

/// A glyph quad of a layer drawn in place under a turn, with the turn that
/// places it: glyphs of layers turned differently then share a draw, and
/// every other glyph keeps [`GlyphInstance`]'s smaller layout.
#[repr(C)]
#[derive(Copy, Clone, Debug, PartialEq, Pod, Zeroable)]
pub(crate) struct TurnedGlyph {
    glyph: GlyphInstance,
    /// The turn, row-major.
    turn: [f32; 4],
    /// The turn's translation in `xy`.
    translation: [f32; 4],
}

impl TurnedGlyph {
    const ATTRIBS: [wgpu::VertexAttribute; 6] = wgpu::vertex_attr_array![
        0 => Float32x4,
        1 => Float32x4,
        2 => Float32x4,
        3 => Float32x4,
        4 => Float32x4,
        5 => Float32x4
    ];

    pub(crate) fn new(glyph: GlyphInstance, turn: SegmentTransform) -> Self {
        let (linear, [x, y], _) = turn.uniform_parts();
        Self {
            glyph,
            turn: linear,
            translation: [x, y, 0.0, 0.0],
        }
    }

    fn desc() -> wgpu::VertexBufferLayout<'static> {
        wgpu::VertexBufferLayout {
            array_stride: std::mem::size_of::<TurnedGlyph>() as wgpu::BufferAddress,
            step_mode: wgpu::VertexStepMode::Instance,
            attributes: &Self::ATTRIBS,
        }
    }
}

/// The glyph quads a pass draws from its shared buffers: the plain ones and
/// those of layers drawn in place under a turn.
#[derive(Default)]
pub(crate) struct GlyphInstances {
    pub(crate) plain: FrameScratch<GlyphInstance>,
    pub(crate) turned: FrameScratch<TurnedGlyph>,
}

impl GlyphInstances {
    pub(crate) fn clear(&mut self) {
        self.plain.clear();
        self.turned.clear();
    }

    /// Ends a frame: see [`FrameScratch::end_frame`]. Passes need the
    /// most glyphs at startup, when nothing is cached yet.
    pub(crate) fn end_frame(&mut self) {
        self.plain.end_frame();
        self.turned.end_frame();
    }

    fn lens(&self) -> (usize, usize) {
        (self.plain.len(), self.turned.len())
    }

    /// How many turned glyphs, or plain ones, the pass holds.
    fn len_of(&self, turned: bool) -> usize {
        if turned {
            self.turned.len()
        } else {
            self.plain.len()
        }
    }

    fn truncate(&mut self, (plain, turned): (usize, usize)) {
        self.plain.truncate(plain);
        self.turned.truncate(turned);
    }
}

/// A text clip's edges in device pixels of the space its glyph quads are
/// laid out in, before a viewport transform moves them.
fn glyph_clip_edges(clip: Rect, root_scale: f32) -> [f32; 4] {
    [
        canonicalize_device_coordinate(clip.x * root_scale),
        canonicalize_device_coordinate(clip.y * root_scale),
        canonicalize_device_coordinate((clip.x + clip.width) * root_scale),
        canonicalize_device_coordinate((clip.y + clip.height) * root_scale),
    ]
}

/// Where the glyph quads of a text are cut, in the quads' own space. Under a
/// turned viewport the cut is the text's `clip` itself, before the turn,
/// which no scissor can follow; the text then draws without one. Otherwise
/// it is the pixel edges of the text's `scissor`, its draw rect within its
/// clip: cut there, the quads cover exactly the pixels the scissor would
/// pass, with the same texels, so the text needs no scissor of its own and
/// draws in one call with its neighbours, even where a last line's
/// descenders or a glyph's padding reach past its draw rect.
fn glyph_cut_edges(
    clip: Option<Rect>,
    scissor: TargetRect,
    viewport: ViewportUniformParams,
    root_scale: f32,
) -> Option<[f32; 4]> {
    if !viewport.transform.is_identity() {
        return clip.map(|clip| glyph_clip_edges(clip, root_scale));
    }
    let (x, y, width, height) = scissor;
    let [offset_x, offset_y] = viewport.offset;
    Some([
        x as f32 + offset_x,
        y as f32 + offset_y,
        (x + width) as f32 + offset_x,
        (y + height) as f32 + offset_y,
    ])
}

#[repr(C)]
#[derive(Copy, Clone, Debug, Pod, Zeroable)]
struct Uniforms {
    viewport: [f32; 2],
    viewport_offset: [f32; 2],
    transform: [f32; 4],
    translation: [f32; 2],
    reserved: [f32; 2],
    inverse: [f32; 4],
    origin: [f32; 2],
    origin_reserved: [f32; 2],
    placement: PlacementData,
}

impl Uniforms {
    fn of(params: ViewportUniformParams, placement: PlacementData, tier: RunTier) -> Self {
        let (transform, translation, inverse) = params.transform.uniform_parts();
        Self {
            viewport: [params.width as f32, params.height as f32],
            viewport_offset: params.offset,
            transform,
            translation,
            reserved: [params.depth_base, f32::from(tier as u8)],
            inverse,
            origin: params.origin,
            origin_reserved: [0.0; 2],
            placement,
        }
    }
}

static SURVIVE_GPU_ERRORS: DebugToggle = DebugToggle::new("CRANPOSE_SURVIVE_GPU_ERRORS");

fn survive_gpu_errors_enabled() -> bool {
    !SURVIVE_GPU_ERRORS.equals("0")
}

struct CachedImageTexture {
    bind_groups: ImageBindGroups,
    _texture: wgpu::Texture,
    _view: wgpu::TextureView,
    bytes: usize,
}

struct ImageBindGroups {
    alpha_mask: bool,
    nearest: wgpu::BindGroup,
    linear: wgpu::BindGroup,
}

impl ImageBindGroups {
    fn binding(&self, sampling: ImageSampling) -> ImageBinding {
        ImageBinding {
            bind_group: match sampling {
                ImageSampling::Nearest => self.nearest.clone(),
                ImageSampling::Linear => self.linear.clone(),
            },
            alpha_mask: self.alpha_mask,
        }
    }
}

/// What an image's draw binds: its texture's bind group for the draw's
/// sampling, which keeps the texture alive whatever the cache evicts before
/// the pass is drawn, and whether the texture is an alpha mask.
pub(crate) struct ImageBinding {
    bind_group: wgpu::BindGroup,
    alpha_mask: bool,
}

#[derive(Clone, Copy)]
struct GlyphAtlasEntry {
    x: u16,
    y: u16,
    width: u16,
    height: u16,
}

fn next_glyph_atlas_size(current: u32, max: u32) -> u32 {
    current.saturating_mul(2).clamp(1, max.max(1))
}

/// The samplers the glyph atlas binds: nearest for glyphs on the pixel grid,
/// linear for glyphs a segment transform turns.
#[derive(Clone, Copy)]
struct GlyphSamplers<'a> {
    nearest: &'a wgpu::Sampler,
    linear: &'a wgpu::Sampler,
}

/// A glyph's atlas slot: its mask corrected for the luminance class of the
/// text it is drawn in, so one glyph in light and dark text takes two slots.
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
struct GlyphAtlasSlotKey {
    glyph: SoftwareGlyphAtlasKey,
    luminance: TextLuminance,
}

impl GlyphAtlasSlotKey {
    fn new(glyph: SoftwareGlyphAtlasKey, color: Color) -> Self {
        Self {
            glyph,
            luminance: TextLuminance::of_color(color),
        }
    }
}

/// Slots of [`RunAtlasEntries`], by glyph id.
const RUN_ATLAS_ENTRY_SLOTS: usize = 64;

/// The atlas entries a run's preparation has looked up, by glyph id: a run
/// repeats its letters, and each atlas lookup hashes the slot key and moves
/// the entry up the atlas's recency order, where the first lookup already
/// put it for this frame.
struct RunAtlasEntries {
    slots: [Option<(GlyphAtlasSlotKey, GlyphAtlasEntry)>; RUN_ATLAS_ENTRY_SLOTS],
}

impl RunAtlasEntries {
    fn new() -> Self {
        Self {
            slots: [None; RUN_ATLAS_ENTRY_SLOTS],
        }
    }

    /// `glyph`'s entry from an earlier glyph of the run, else from
    /// `look_up`, remembered for the rest of it.
    fn entry(
        &mut self,
        renderer: &mut GpuRenderer,
        glyph: &SoftwareGlyphAtlasPlacement,
        look_up: impl FnOnce(&mut GpuRenderer) -> Result<GlyphAtlasEntry, String>,
    ) -> Result<GlyphAtlasEntry, String> {
        let key = GlyphAtlasSlotKey::new(glyph.key, glyph.color);
        let slot = &mut self.slots[glyph.key.glyph_id as usize % RUN_ATLAS_ENTRY_SLOTS];
        if let Some((seen, entry)) = *slot
            && seen == key
        {
            renderer.frame_stats.record_text_glyph_atlas_hits(1);
            return Ok(entry);
        }
        let entry = look_up(renderer)?;
        *slot = Some((key, entry));
        Ok(entry)
    }
}

struct TextGlyphAtlas {
    texture: wgpu::Texture,
    _view: wgpu::TextureView,
    texel_bind_group: Rc<wgpu::BindGroup>,
    filtered_bind_group: Rc<wgpu::BindGroup>,
    entries: BoundedLruCache<GlyphAtlasSlotKey, GlyphAtlasEntry>,
    generation: u64,
    size: u32,
    max_size: u32,
    cursor_x: u32,
    cursor_y: u32,
    row_height: u32,
    /// The glyphs placed since [`Self::flush_uploads`], in the order they
    /// were placed, and their coverage one after another.
    pending: Vec<GlyphAtlasEntry>,
    pending_coverage: Vec<u8>,
    upload_scratch: Vec<u8>,
}

impl TextGlyphAtlas {
    fn new(
        device: &wgpu::Device,
        image_layout: &wgpu::BindGroupLayout,
        samplers: GlyphSamplers<'_>,
        size: u32,
    ) -> Self {
        // The side stays a power of two, whose texel is exact: see
        // [`TextGlyphAtlas::texel`].
        let device_max = device.limits().max_texture_dimension_2d.max(1);
        let max_size = TEXT_GLYPH_ATLAS_MAX_SIZE.min(1 << device_max.ilog2());
        let size = size
            .next_power_of_two()
            .clamp(TEXT_GLYPH_ATLAS_MIN_SIZE.min(max_size), max_size);
        let texture = Self::create_texture(device, size);
        let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
        let bind = |label: &'static str, sampler: &wgpu::Sampler| {
            Rc::new(device.create_bind_group(&wgpu::BindGroupDescriptor {
                label: Some(label),
                layout: image_layout,
                entries: &[
                    wgpu::BindGroupEntry {
                        binding: 0,
                        resource: wgpu::BindingResource::TextureView(&view),
                    },
                    wgpu::BindGroupEntry {
                        binding: 1,
                        resource: wgpu::BindingResource::Sampler(sampler),
                    },
                ],
            }))
        };
        let texel_bind_group = bind("Text Glyph Atlas Bind Group", samplers.nearest);
        let filtered_bind_group = bind("Filtered Text Glyph Atlas Bind Group", samplers.linear);
        Self {
            texture,
            _view: view,
            texel_bind_group,
            filtered_bind_group,
            entries: BoundedLruCache::with_capacity_at_least_one(MAX_TEXT_GLYPH_ATLAS_ITEMS),
            generation: 0,
            size,
            max_size,
            cursor_x: TEXT_GLYPH_ATLAS_PADDING,
            cursor_y: TEXT_GLYPH_ATLAS_PADDING,
            row_height: 0,
            pending: Vec::new(),
            pending_coverage: Vec::new(),
            upload_scratch: Vec::new(),
        }
    }

    fn create_texture(device: &wgpu::Device, size: u32) -> wgpu::Texture {
        device.create_texture(&wgpu::TextureDescriptor {
            label: Some("Text Glyph Atlas Texture"),
            size: wgpu::Extent3d {
                width: size,
                height: size,
                depth_or_array_layers: 1,
            },
            mip_level_count: 1,
            sample_count: 1,
            dimension: wgpu::TextureDimension::D2,
            format: wgpu::TextureFormat::R8Unorm,
            usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
            view_formats: &[],
        })
    }

    fn reset(
        &mut self,
        device: &wgpu::Device,
        image_layout: &wgpu::BindGroupLayout,
        samplers: GlyphSamplers<'_>,
    ) {
        let generation = self.generation.wrapping_add(1);
        let grown = next_glyph_atlas_size(self.size, self.max_size);
        let mut next = Self::new(device, image_layout, samplers, grown);
        next.generation = generation;
        *self = next;
    }

    /// The atlas bound for glyphs drawn under `transform`: texel for texel
    /// on the pixel grid, filtered once a transform turns the glyph quads
    /// off it.
    fn bind_group(&self, transform: SegmentTransform) -> Rc<wgpu::BindGroup> {
        Rc::clone(if transform.is_identity() {
            &self.texel_bind_group
        } else {
            &self.filtered_bind_group
        })
    }

    fn generation(&self) -> u64 {
        self.generation
    }

    fn size(&self) -> u32 {
        self.size
    }

    /// One texel's share of the atlas side. The side is a power of two, so
    /// this reciprocal is exact and a coordinate times it is the coordinate
    /// divided by the side: a glyph's quad takes eight of them.
    fn texel(&self) -> f32 {
        (self.size() as f32).recip()
    }

    fn entry(&mut self, key: &GlyphAtlasSlotKey) -> Option<GlyphAtlasEntry> {
        self.entries.get(key).copied()
    }

    fn allocate(&mut self, width: u32, height: u32) -> Option<GlyphAtlasEntry> {
        let max_extent = self.size.saturating_sub(TEXT_GLYPH_ATLAS_PADDING * 2);
        if width == 0 || height == 0 || width > max_extent || height > max_extent {
            return None;
        }

        if self.cursor_x + width + TEXT_GLYPH_ATLAS_PADDING > self.size {
            self.cursor_x = TEXT_GLYPH_ATLAS_PADDING;
            self.cursor_y = self
                .cursor_y
                .saturating_add(self.row_height)
                .saturating_add(TEXT_GLYPH_ATLAS_PADDING);
            self.row_height = 0;
        }
        if self.cursor_y > self.size - height - TEXT_GLYPH_ATLAS_PADDING {
            return None;
        }

        let entry = GlyphAtlasEntry {
            x: self.cursor_x as u16,
            y: self.cursor_y as u16,
            width: width as u16,
            height: height as u16,
        };
        self.cursor_x = self
            .cursor_x
            .saturating_add(width)
            .saturating_add(TEXT_GLYPH_ATLAS_PADDING);
        self.row_height = self.row_height.max(height);
        Some(entry)
    }

    /// The atlas entry of `glyph`, placed now if the atlas does not hold it
    /// yet; a placed glyph reaches the texture at [`Self::flush_uploads`].
    /// `None` when the atlas is full.
    fn upload_glyph(
        &mut self,
        glyph: &SoftwareGlyphAtlasGlyph,
        frame_stats: &mut gpu_stats::FrameStats,
    ) -> Option<GlyphAtlasEntry> {
        let key = GlyphAtlasSlotKey::new(glyph.key, glyph.color);
        if let Some(entry) = self.entry(&key) {
            frame_stats.record_text_glyph_atlas_hits(1);
            return Some(entry);
        }

        let width = u32::try_from(glyph.mask.width).ok()?;
        let height = u32::try_from(glyph.mask.height).ok()?;
        let entry = self.allocate(width, height)?;
        let correction = key.luminance.correction();
        self.pending_coverage.extend(
            glyph
                .mask
                .alpha
                .iter()
                .map(|&coverage| correction.apply(coverage)),
        );
        self.pending.push(entry);
        frame_stats.record_text_glyph_atlas_miss(width, height);
        self.entries.put(key, entry);
        Some(entry)
    }

    /// Writes the glyphs placed since the last call to the texture, one
    /// write for each run of them placed side by side on a row. The atlas
    /// never places a glyph where one was, so the run's padding and the
    /// rows under its shorter glyphs are empty, and the write leaves them
    /// so. A frame calls this before it submits draws that read them.
    fn flush_uploads(
        &mut self,
        queue: &wgpu::Queue,
        executor: &mut WgpuFrameGraphExecutor,
        frame_stats: &mut gpu_stats::FrameStats,
    ) {
        let mut coverage = 0;
        let mut run_start = 0;
        while run_start < self.pending.len() {
            let first = self.pending[run_start];
            let mut run_end = run_start + 1;
            while self
                .pending
                .get(run_end)
                .is_some_and(|next| next.y == first.y && next.x > self.pending[run_end - 1].x)
            {
                run_end += 1;
            }
            let run = &self.pending[run_start..run_end];
            let last = run[run.len() - 1];
            let x = u32::from(first.x);
            let width = u32::from(last.x) + u32::from(last.width) - x;
            let height = run
                .iter()
                .map(|entry| u32::from(entry.height))
                .max()
                .unwrap_or(0);
            self.upload_scratch.clear();
            self.upload_scratch.resize((width * height) as usize, 0);
            for entry in run {
                let glyph_width = usize::from(entry.width);
                let left = usize::from(entry.x) - x as usize;
                for row in 0..usize::from(entry.height) {
                    let source = coverage + row * glyph_width;
                    let target = row * width as usize + left;
                    self.upload_scratch[target..target + glyph_width]
                        .copy_from_slice(&self.pending_coverage[source..source + glyph_width]);
                }
                coverage += glyph_width * usize::from(entry.height);
            }
            let upload_stats = executor.upload_texture(
                queue,
                wgpu::TexelCopyTextureInfo {
                    texture: &self.texture,
                    mip_level: 0,
                    origin: wgpu::Origin3d {
                        x,
                        y: u32::from(first.y),
                        z: 0,
                    },
                    aspect: wgpu::TextureAspect::All,
                },
                &self.upload_scratch,
                wgpu::TexelCopyBufferLayout {
                    offset: 0,
                    bytes_per_row: Some(width),
                    rows_per_image: Some(height),
                },
                wgpu::Extent3d {
                    width,
                    height,
                    depth_or_array_layers: 1,
                },
            );
            frame_stats.record_command_stats(upload_stats);
            run_start = run_end;
        }
        self.pending.clear();
        self.pending_coverage.clear();
    }
}

/// The image pipeline a draw takes: in a pass with depth or not, for an
/// alpha mask or a color image, and taking a rounded clip in place or not.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct ImageVariant {
    depth: bool,
    alpha_mask: bool,
    rounded: bool,
}

pub(crate) struct ImageDrawCmd {
    index_start: u32,
    scissor: (u32, u32, u32, u32),
    image: ImageBinding,
    /// The rounded clip whose coverage the image takes in place: its index
    /// among the pass's clip instances.
    clip: Option<u32>,
}

impl ImageVariant {
    /// The variant `cmd` draws with in a pass with a depth buffer or not.
    fn of(cmd: &ImageDrawCmd, depth: bool) -> Self {
        Self {
            depth,
            alpha_mask: cmd.image.alpha_mask,
            rounded: cmd.clip.is_some(),
        }
    }
}

impl ImageDrawCmd {
    /// Target pixels the draw can touch: its scissor.
    pub(crate) fn bounds(&self) -> TargetRect {
        self.scissor
    }
}

/// Which glyph pipeline draws a stretch of quads: plain quads, sampled
/// within their glyph's texel centers; plain quads on whole pixels at one
/// texel a pixel, whose samples land on those centers unclamped; or quads
/// of a layer drawn in place under a turn, which are [`TurnedGlyph`]s.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum GlyphQuads {
    Plain,
    Aligned,
    Turned,
}

impl GlyphQuads {
    fn turned(self) -> bool {
        self == Self::Turned
    }

    /// The plain kind that fits quads whose every corner is `aligned`.
    fn plain(aligned: bool) -> Self {
        if aligned { Self::Aligned } else { Self::Plain }
    }
}

/// Whether a plain glyph quad at `rect` lands on whole pixels of a target
/// whose origin sits at `offset`: its quads hold one texel a pixel.
fn glyph_rect_aligned(rect: [f32; 4], offset: [f32; 2]) -> bool {
    let whole = |value: f32| value.fract() == 0.0;
    whole(rect[0] - offset[0])
        && whole(rect[1] - offset[1])
        && whole(rect[2] - offset[0])
        && whole(rect[3] - offset[1])
}

#[derive(Clone)]
enum GlyphDrawSource {
    Shared {
        instance_start: u32,
        instance_count: u32,
        quads: GlyphQuads,
    },
    Retained {
        run: Rc<CachedGpuTextGlyphRun>,
        uniform_slot: usize,
        aligned: bool,
    },
}

#[derive(Clone)]
pub(crate) struct GlyphDrawCmd {
    atlas: Rc<wgpu::BindGroup>,
    source: GlyphDrawSource,
    /// The scissor the text's own clip needs, `None` when its quads lie
    /// inside that clip anyway and only the batch's bound applies.
    scissor: Option<(u32, u32, u32, u32)>,
    /// Target pixels the draw can touch, for ordering it against others.
    bounds: (u32, u32, u32, u32),
}

/// One draw of a glyph batch: a stretch of shared quads, or a retained run.
struct GlyphDraw<'a> {
    atlas: &'a Rc<wgpu::BindGroup>,
    scissor: Option<(u32, u32, u32, u32)>,
    step: GlyphDrawStep<'a>,
}

enum GlyphDrawStep<'a> {
    /// A stretch of the shared instances of one kind.
    Shared(std::ops::Range<u32>, GlyphQuads),
    Retained {
        run: &'a CachedGpuTextGlyphRun,
        uniform_slot: usize,
        aligned: bool,
    },
}

/// The draws a batch's glyph commands take: consecutive shared commands with
/// one atlas and one scissor whose quads follow each other in the frame's
/// shared buffer draw as one.
struct GlyphDraws<'a> {
    cmds: &'a [GlyphDrawCmd],
}

impl<'a> GlyphDraws<'a> {
    fn new(cmds: &'a [GlyphDrawCmd]) -> Self {
        Self { cmds }
    }
}

impl<'a> Iterator for GlyphDraws<'a> {
    type Item = GlyphDraw<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        let (first, rest) = self.cmds.split_first()?;
        self.cmds = rest;
        let step = match &first.source {
            GlyphDrawSource::Retained {
                run,
                uniform_slot,
                aligned,
            } => GlyphDrawStep::Retained {
                run,
                uniform_slot: *uniform_slot,
                aligned: *aligned,
            },
            GlyphDrawSource::Shared {
                instance_start,
                instance_count,
                quads,
            } => {
                let mut end = instance_start + instance_count;
                while let Some((next, rest)) = self.cmds.split_first() {
                    match next.source {
                        GlyphDrawSource::Shared {
                            instance_start,
                            instance_count,
                            quads: next_quads,
                        } if instance_start == end
                            && next_quads == *quads
                            && next.scissor == first.scissor
                            && Rc::ptr_eq(&next.atlas, &first.atlas) =>
                        {
                            end += instance_count;
                            self.cmds = rest;
                        }
                        _ => break,
                    }
                }
                GlyphDrawStep::Shared(*instance_start..end, *quads)
            }
        };
        Some(GlyphDraw {
            atlas: &first.atlas,
            scissor: first.scissor,
            step,
        })
    }
}

impl GlyphDrawCmd {
    /// The quads the command draws, which pick its pipeline.
    fn quads(&self) -> GlyphQuads {
        match &self.source {
            GlyphDrawSource::Shared { quads, .. } => *quads,
            GlyphDrawSource::Retained { aligned, .. } => GlyphQuads::plain(*aligned),
        }
    }

    fn shared(
        (instances, quads): (std::ops::Range<usize>, GlyphQuads),
        scissor: Option<(u32, u32, u32, u32)>,
        bounds: (u32, u32, u32, u32),
        atlas: Rc<wgpu::BindGroup>,
    ) -> Self {
        let start = u32::try_from(instances.start).unwrap_or(u32::MAX);
        let end = u32::try_from(instances.end).unwrap_or(u32::MAX);
        Self {
            atlas,
            source: GlyphDrawSource::Shared {
                instance_start: start,
                instance_count: end.saturating_sub(start),
                quads,
            },
            scissor,
            bounds,
        }
    }

    fn retained(
        (run, uniform_slot, aligned): (Rc<CachedGpuTextGlyphRun>, usize, bool),
        scissor: (u32, u32, u32, u32),
        atlas: Rc<wgpu::BindGroup>,
    ) -> Self {
        Self {
            atlas,
            source: GlyphDrawSource::Retained {
                run,
                uniform_slot,
                aligned,
            },
            scissor: Some(scissor),
            bounds: scissor,
        }
    }

    /// Target pixels the draw can touch.
    pub(crate) fn bounds(&self) -> (u32, u32, u32, u32) {
        self.bounds
    }

    /// Whether the draw's quads are [`TurnedGlyph`]s, or `None` for a
    /// retained run, which draws on its own.
    fn turned(&self) -> Option<bool> {
        match self.source {
            GlyphDrawSource::Shared { quads, .. } => Some(quads.turned()),
            GlyphDrawSource::Retained { .. } => None,
        }
    }
}

/// The kind of glyph draw, turned (`true`) or plain, that can move after
/// every draw of the other kind in `draws` without moving past a draw it
/// shares a pixel with: glyphs blend, so only draws that share pixels must
/// keep their order. The fewer kind is tried first, which checks fewer
/// pairs. `None` when neither can move, a retained run is among them, or
/// the kinds do not change back and forth.
pub(crate) fn movable_glyph_kind<T>(
    draws: &[T],
    kind: impl Fn(&T) -> Option<bool>,
    bounds: impl Fn(&T) -> TargetRect,
) -> Option<bool> {
    let mut turned = 0;
    for draw in draws {
        turned += usize::from(kind(draw)?);
    }
    let changes = draws
        .windows(2)
        .filter(|pair| kind(&pair[0]) != kind(&pair[1]))
        .count();
    if changes < 2 {
        return None;
    }
    let fewer = turned * 2 < draws.len();
    [fewer, !fewer].into_iter().find(|&moved| {
        draws.iter().enumerate().all(|(index, draw)| {
            kind(draw) != Some(moved)
                || draws[index + 1..].iter().all(|later| {
                    kind(later) == Some(moved)
                        || !crate::draw_pass::target_rects_overlap(bounds(draw), bounds(later))
                })
        })
    })
}

/// Moves a glyph batch's retained runs after its shared quads, each in its
/// order, when no run shares a pixel with a shared draw after it: the
/// shared quads around a run, which follow each other in the frame's
/// buffer, then draw at once. A card's body text, a retained run between
/// its title and its labels, split them into two draws. Returns how many
/// shared draws lead the batch, none when its runs keep their places;
/// `moved` is scratch.
pub(crate) fn defer_retained_glyph_runs(
    cmds: &mut [GlyphDrawCmd],
    moved: &mut Vec<GlyphDrawCmd>,
) -> usize {
    let retained = |cmd: &GlyphDrawCmd| cmd.turned().is_none();
    let shared = cmds.iter().filter(|cmd| !retained(cmd)).count();
    if cmds[..shared].iter().all(|cmd| !retained(cmd)) {
        return shared;
    }
    let deferrable = cmds.iter().enumerate().all(|(index, cmd)| {
        !retained(cmd)
            || cmds[index + 1..].iter().all(|later| {
                retained(later)
                    || !crate::draw_pass::target_rects_overlap(cmd.bounds(), later.bounds())
            })
    });
    if !deferrable {
        return 0;
    }
    moved.clear();
    let mut kept = 0;
    for read in 0..cmds.len() {
        if retained(&cmds[read]) {
            moved.push(cmds[read].clone());
        } else {
            cmds.swap(kept, read);
            kept += 1;
        }
    }
    for (slot, cmd) in cmds[kept..].iter_mut().zip(moved.drain(..)) {
        *slot = cmd;
    }
    shared
}

/// Moves a glyph batch's draws of the kind [`movable_glyph_kind`] picks
/// after those of the other kind, each kind in its order, so each draws
/// once instead of once per change between them. `moved` is scratch.
pub(crate) fn group_glyph_kinds(cmds: &mut [GlyphDrawCmd], moved: &mut Vec<GlyphDrawCmd>) {
    let Some(kind) = movable_glyph_kind(cmds, GlyphDrawCmd::turned, GlyphDrawCmd::bounds) else {
        return;
    };
    moved.clear();
    let mut kept = 0;
    for read in 0..cmds.len() {
        if cmds[read].turned() == Some(kind) {
            moved.push(cmds[read].clone());
        } else {
            cmds.swap(kept, read);
            kept += 1;
        }
    }
    for (slot, cmd) in cmds[kept..].iter_mut().zip(moved.drain(..)) {
        *slot = cmd;
    }
}

#[derive(Clone, Copy, Debug, PartialEq)]
struct ImageUvRect {
    min: [f32; 2],
    max: [f32; 2],
    sample_bounds: [f32; 4],
}

/// A growable vertex buffer and index buffer pair.
/// One pass's image and shared glyph quads: the frame's vertex and index
/// uploads they were appended to.
pub(crate) struct ImageSlot {
    vertices: BufferUpload,
    indices: BufferUpload,
    /// The rounded clips the pass's images take in place, one instance each.
    clips: Option<BufferUpload>,
}

fn image_vertex_spec() -> UploadAllocatorSpec {
    UploadAllocatorSpec::vertex("Image Vertex Buffer", std::mem::size_of::<Vertex>() as u64)
}

fn image_index_spec() -> UploadAllocatorSpec {
    UploadAllocatorSpec::index("Image Index Buffer", std::mem::size_of::<u32>() as u64)
}

fn glyph_instance_spec() -> UploadAllocatorSpec {
    UploadAllocatorSpec::vertex(
        "Glyph Instance Buffer",
        std::mem::size_of::<GlyphInstance>() as u64,
    )
}

fn turned_glyph_spec() -> UploadAllocatorSpec {
    UploadAllocatorSpec::vertex(
        "Turned Glyph Instance Buffer",
        std::mem::size_of::<TurnedGlyph>() as u64,
    )
}

#[derive(Default)]
struct ViewportUniforms {
    uploads: FrameUploadAllocators,
    slots: Vec<UniformUpload>,
}

impl ViewportUniforms {
    fn begin_frame(&mut self) {
        self.slots.clear();
        self.uploads.reset();
    }

    fn claim(
        &mut self,
        device: &wgpu::Device,
        layout: &wgpu::BindGroupLayout,
        uniforms: &Uniforms,
    ) -> usize {
        let slot = self.slots.len();
        self.slots.push(self.uploads.upload_uniform(
            UploadAllocatorId::Viewport,
            UploadAllocatorSpec::uniform(
                "Viewport Uniform Buffer",
                "Viewport Uniform Bind Group",
                std::mem::size_of::<Uniforms>() as u64,
            ),
            device,
            layout,
            bytemuck::bytes_of(uniforms),
        ));
        slot
    }

    fn bind(&self, pass: &mut TrackedPass<'_, '_>, slot: usize) -> Result<(), String> {
        let uniform = self
            .slots
            .get(slot)
            .ok_or_else(|| "viewport uniform slot was never claimed this frame".to_string())?;
        pass.set_bind_group(0, &uniform.bind_group, &[uniform.offset]);
        Ok(())
    }

    fn stage_pending(
        &mut self,
        device: &wgpu::Device,
        recorder: &mut impl FrameCommandRecorder,
    ) -> FrameCommandStats {
        self.uploads.stage_pending(|buffer, offset, bytes| {
            recorder.stage_frame_buffer_copy(device, buffer, offset, bytes)
        })
    }
}

#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) struct ViewportUniformParams {
    pub(crate) width: u32,
    pub(crate) height: u32,
    pub(crate) offset: [f32; 2],
    pub(crate) transform: SegmentTransform,
    /// Where the drawn vertices' origin sits in the segment's device space,
    /// added before the transform: zero, except for a retained glyph run
    /// drawn under a transform, whose vertices sit at its raster origin. The
    /// sum is the one the shared path writes, so both draw the same pixels.
    pub(crate) origin: [f32; 2],
    /// The pass-order index of the batch's first shape record, from which
    /// each record's depth counts.
    pub(crate) depth_base: f32,
}

impl ViewportUniformParams {
    /// The logical rect of the drawn scene that the target shows.
    pub(crate) fn scene_rect(self, root_scale: f32) -> Rect {
        segment_scene_rect(
            self.transform,
            Rect {
                x: self.offset[0],
                y: self.offset[1],
                width: self.width as f32,
                height: self.height as f32,
            },
            root_scale,
        )
    }
}

/// The logical rect of a scene drawn under `transform` that the device rect
/// `device` of the target's scene space shows: the rect itself when the
/// scene is not transformed, else the bounds it maps back to.
pub(crate) fn segment_scene_rect(
    transform: SegmentTransform,
    device: Rect,
    root_scale: f32,
) -> Rect {
    let device = if transform.is_identity() {
        device
    } else {
        transform.segment_bounds(device)
    };
    Rect {
        x: device.x / root_scale,
        y: device.y / root_scale,
        width: device.width / root_scale,
        height: device.height / root_scale,
    }
}

/// A stored run's draws for one pass: the tables current when it was
/// prepared, the uniform slot holding its placement, and the range of the
/// pass's run draws with the pipeline and vertex range of each segment's
/// quads and bands.
pub(crate) struct StoreRunBatch {
    pub(crate) tables: StoredTables,
    pub(crate) uniform_slot: usize,
    pub(crate) draws: std::ops::Range<usize>,
    /// The scissor an unturned run's clip puts on its paint: see
    /// [`clip_scissor`].
    pub(crate) clip: Option<TargetRect>,
}

struct CompositionTarget {
    target: Rc<OffscreenTarget>,
    output_bind_group: wgpu::BindGroup,
}

/// Where a frame renders: straight into the presentable image, or into the
/// reusable composition target that the output conversion then copies out.
enum FrameRoot {
    Surface(Rc<OffscreenTarget>),
    Composition(CompositionTarget),
}

impl FrameRoot {
    fn target(&self) -> &Rc<OffscreenTarget> {
        match self {
            Self::Surface(target) => target,
            Self::Composition(composition) => &composition.target,
        }
    }

    /// The output conversion's destination and source bind group; nothing
    /// when the frame already rendered into the presentable image.
    fn output<'a>(
        &'a self,
        output_view: Option<&'a wgpu::TextureView>,
        screenshot_bind_group: Option<&'a wgpu::BindGroup>,
    ) -> Option<(&'a wgpu::TextureView, &'a wgpu::BindGroup)> {
        match self {
            Self::Surface(_) => None,
            Self::Composition(composition) => output_view.map(|view| {
                (
                    view,
                    screenshot_bind_group.unwrap_or(&composition.output_bind_group),
                )
            }),
        }
    }
}

/// What a presentable image must allow to be the frame's root target:
/// rendering, and the copies a capture and the opaque-prefix cache take of
/// the page and put back.
const DIRECT_SURFACE_ROOT_USAGES: wgpu::TextureUsages = wgpu::TextureUsages::RENDER_ATTACHMENT
    .union(wgpu::TextureUsages::COPY_SRC)
    .union(wgpu::TextureUsages::COPY_DST);

/// The usages to ask of a presentable image: rendering, and whichever of
/// the copies and sampling the surface offers. Sampling lets a frame whose
/// backdrops read the page draw straight into the image too.
pub fn presentable_root_usages(supported: wgpu::TextureUsages) -> wgpu::TextureUsages {
    wgpu::TextureUsages::RENDER_ATTACHMENT
        | (supported & (DIRECT_SURFACE_ROOT_USAGES | wgpu::TextureUsages::TEXTURE_BINDING))
}

/// Whether the presented image can be the frame's root target: its bytes
/// are the composition format (so the 8-bit output conversion would be an
/// identity), it is the viewport's size, it can be copied as the page is,
/// and it can be sampled when the frame `reads_page` through a backdrop.
/// Metal's images cannot be sampled, so only a frame without backdrops
/// draws straight into one there.
fn surface_is_direct_root(
    texture: &wgpu::Texture,
    composition_format: wgpu::TextureFormat,
    viewport: (u32, u32),
    reads_page: bool,
) -> bool {
    let usage = texture.usage();
    texture.format().remove_srgb_suffix() == composition_format
        && usage.contains(DIRECT_SURFACE_ROOT_USAGES)
        && (!reads_page || usage.contains(wgpu::TextureUsages::TEXTURE_BINDING))
        && (texture.width(), texture.height()) == viewport
}

#[derive(Clone, Copy)]
enum OutputMode {
    Display,
    Screenshot,
}

pub struct GpuRenderer {
    pub(crate) device: Arc<wgpu::Device>,
    pub(crate) queue: Arc<wgpu::Queue>,
    device_errors: Arc<DeviceErrorSentry>,
    renderer_epoch: u64,
    pub(crate) composition_format: wgpu::TextureFormat,
    #[cfg(not(target_arch = "wasm32"))]
    display_format: wgpu::TextureFormat,
    composition_target: Option<CompositionTarget>,
    output_converter: OutputConverter,
    screenshot_converter: OutputConverter,
    adapter_backend: wgpu::Backend,
    pipeline_cache: Option<wgpu::PipelineCache>,
    #[cfg(not(target_arch = "wasm32"))]
    _pipeline_cache_watcher: Option<crate::pipeline_disk_cache::PersistWatcher>,
    /// The pipelines recent launches of another build drew with, built once
    /// this renderer's first frame is drawn.
    #[cfg(not(target_arch = "wasm32"))]
    recent_after_update: Option<Arc<crate::pipeline_records::PipelineRecords>>,
    /// What this renderer draws with, for the next launches.
    recorder: crate::pipeline_recorder::PipelineRecorder,
    /// The background compiler, which builds a pass's missing pipelines side
    /// by side.
    pipeline_compiler: PipelineCompiler,
    /// Where the background compiler says a pipeline a placeholder waited
    /// for landed.
    #[cfg(not(target_arch = "wasm32"))]
    landing: Option<Arc<crate::pipeline_compiler::Landing>>,
    /// The pipelines the background threads had built when this frame
    /// started.
    #[cfg(not(target_arch = "wasm32"))]
    built_at_frame_start: u64,
    /// Whether this frame drew a placeholder in place of an effect whose
    /// pipelines were compiling.
    pub(crate) drew_placeholder: bool,
    shape_pipelines: ShapePipelines,
    /// Image and glyph pipelines for passes without and with a depth buffer.
    image_pipeline: [FixedPipeline; 8],
    image_pipeline_dst_out: [FixedPipeline; 8],
    /// Indexed by depth, then turned: see [`GpuRenderer::glyph_atlas_pipeline`].
    glyph_atlas_pipeline: [FixedPipeline; 6],
    /// Round rect shadow pipelines without and with a depth buffer.
    rrect_shadow_pipeline: [FixedPipeline; 2],
    image_shader: SharedShader,
    glyph_atlas_shader: SharedShader,
    rrect_shadow_shader: SharedShader,
    /// Transient depth buffers by target size, for passes that lay opaque
    /// interiors down first, dropped once no frame has used them for
    /// [`crate::idle_pool::IDLE_FRAMES`].
    depth_targets: crate::idle_pool::IdlePool<((u32, u32), wgpu::TextureView)>,
    pub(crate) depth_gate: crate::depth_gate::DepthGate,
    /// The size of the atlas the layer surfaces drawn again go into.
    pub(crate) surface_atlas_sizes: crate::frame::SurfaceAtlasSizes,
    uniform_bind_group_layout: wgpu::BindGroupLayout,
    image_bind_group_layout: wgpu::BindGroupLayout,
    image_nearest_sampler: wgpu::Sampler,
    image_linear_sampler: wgpu::Sampler,
    text_fonts: SoftwareTextFontSet,
    viewport_uniforms: ViewportUniforms,
    run_store: RunStore,
    image_texture_cache: BoundedLruCache<u64, CachedImageTexture>,
    image_texture_cache_bytes: usize,
    external_texture_bindings: BoundedLruCache<u64, ImageBindGroups>,
    text_image_cache: BoundedLruCache<TextImageCacheKey, CachedTextImage>,
    text_glyph_atlas: TextGlyphAtlas,
    text_glyph_run_cache: BoundedLruCache<TextGlyphRunCacheKey, CachedTextGlyphRun>,
    text_glyph_gpu_run_cache: BoundedLruCache<TextGlyphRunCacheKey, Rc<CachedGpuTextGlyphRun>>,
    text_glyph_run_arena: GlyphRunArena,
    /// How run tables and retained glyph runs reach the GPU, and whether
    /// kept layer surfaces stay in their atlas: see [`UploadMode::copy_free`].
    pub(crate) copy_free: UploadMode,
    text_glyph_run_frame: u64,
    /// Text runs the frame has drawn so far, cached or not.
    text_glyph_runs_drawn: usize,
    text_glyph_mask_cache: SoftwareGlyphRasterCache,
    text_line_index_cache: TextLineIndexCache,
    pub(crate) scratch_image_vertices: Vec<Vertex>,
    pub(crate) scratch_image_indices: Vec<u32>,
    pub(crate) scratch_glyph_instances: GlyphInstances,
    pub(crate) scratch_image_cmds: Vec<ImageDrawCmd>,
    pub(crate) scratch_image_clips: Vec<DeviceRoundedClip>,
    pub(crate) scratch_glyph_cmds: Vec<GlyphDrawCmd>,
    pub(crate) scratch_glyph_moved: Vec<GlyphDrawCmd>,
    pub(crate) scratch_shadow_instances: Vec<ShadowInstance>,
    pub(crate) scratch_arena_draws: Vec<RunDrawCall>,
    pub(crate) scratch_pipelines: PipelineDemand,
    pub(crate) scratch_held: crate::draw_pass::HeldDraws,
    scratch_text_glyph_run: Vec<SoftwareGlyphAtlasRunGlyph>,
    scratch_text_glyph_entries: Vec<GlyphAtlasEntry>,
    run_glyph_scratch: RunGlyphScratch,
    frame_graph_executor: WgpuFrameGraphExecutor,
    pub(crate) stage_side_scratch: StageSideScratch,
    deferred_offscreen_releases: Vec<OffscreenTarget>,
    pub(crate) effect_renderer: EffectRenderer,
    pub(crate) layer_cache: LayerCache,
    pub(crate) ablation: Ablation,
    pub(crate) ablation_frames: u32,
    pub(crate) nesting_overflow_reported: bool,
    pub(crate) backdrop_gates: HashMap<NodeId, AdmissionGate>,
    pub(crate) fill_gates: HashMap<DrawCommandId, AdmissionGate>,
    pub(crate) effect_gates: HashMap<NodeId, AdmissionGate>,
    pub(crate) source_gates: HashMap<NodeId, AdmissionGate>,
    transparent_sources: HashMap<(u32, u32), Rc<OffscreenTarget>>,
    shadow_surface_cache: BoundedLruCache<ShadowSurfaceCacheKey, CachedShadowSurface>,
    shadow_surface_cache_bytes: u64,
    pub(crate) frame_stats: gpu_stats::FrameStats,
    last_frame_stats: Option<gpu_stats::FrameStatsSnapshot>,
    pending_frame_warmup_frames: u8,
    frame_count: u64,
    /// How many requested shader warm-ups this renderer has queued.
    shader_warm_ups_queued: usize,
}

/// What a frame clears to before it draws: nothing for a transparent
/// window, the framework's background for every other surface.
pub fn frame_clear_color(transparent: bool) -> wgpu::Color {
    if transparent {
        wgpu::Color::TRANSPARENT
    } else {
        CLEAR_COLOR
    }
}

fn image_sampler_descriptor(sampling: ImageSampling) -> wgpu::SamplerDescriptor<'static> {
    let filter = match sampling {
        ImageSampling::Nearest => wgpu::FilterMode::Nearest,
        ImageSampling::Linear => wgpu::FilterMode::Linear,
    };
    wgpu::SamplerDescriptor {
        label: Some(match sampling {
            ImageSampling::Nearest => "Nearest Image Sampler",
            ImageSampling::Linear => "Linear Image Sampler",
        }),
        address_mode_u: wgpu::AddressMode::ClampToEdge,
        address_mode_v: wgpu::AddressMode::ClampToEdge,
        address_mode_w: wgpu::AddressMode::ClampToEdge,
        mag_filter: filter,
        min_filter: filter,
        mipmap_filter: wgpu::MipmapFilterMode::Nearest,
        ..Default::default()
    }
}

/// What a [`GpuRenderer`] is built from. It crosses to the present thread
/// whole, so it holds only owned, `Send` parts.
pub(crate) struct GpuRendererInit {
    pub(crate) device: Arc<wgpu::Device>,
    pub(crate) queue: Arc<wgpu::Queue>,
    pub(crate) surface_format: wgpu::TextureFormat,
    pub(crate) adapter_backend: wgpu::Backend,
    pub(crate) adapter_downlevel: wgpu::DownlevelFlags,
    pub(crate) text_fonts: SoftwareTextFontSet,
    pub(crate) renderer_epoch: u64,
    pub(crate) pipeline_compilation: PipelineCompilation,
}

impl GpuRenderer {
    pub fn new(init: GpuRendererInit) -> Self {
        let GpuRendererInit {
            device,
            queue,
            surface_format,
            adapter_backend,
            adapter_downlevel,
            text_fonts,
            renderer_epoch,
            pipeline_compilation,
        } = init;
        let display_format = surface_format;
        let construction_started = Instant::now();
        let device_errors = Arc::new(DeviceErrorSentry::default());
        if survive_gpu_errors_enabled() {
            let sentry = Arc::clone(&device_errors);
            device.on_uncaptured_error(Arc::new(move |error| sentry.record(&error)));
        }
        let composition_format = crate::offscreen::composition_format(display_format);
        device.set_device_lost_callback(|reason, message| {
            log::error!("[gpu-device] device lost ({reason:?}): {message}");
        });
        let copy_free = UploadMode::copy_free(&device, adapter_backend);
        let mut run_store = RunStore::new(
            &device,
            RunBufferMode::for_device(&device, adapter_downlevel),
            copy_free,
        );
        let uniform_bind_group_layout =
            device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
                label: Some("Viewport Uniform Bind Group Layout"),
                entries: &[wgpu::BindGroupLayoutEntry {
                    binding: 0,
                    visibility: wgpu::ShaderStages::VERTEX_FRAGMENT,
                    ty: wgpu::BindingType::Buffer {
                        ty: wgpu::BufferBindingType::Uniform,
                        has_dynamic_offset: true,
                        min_binding_size: wgpu::BufferSize::new(
                            std::mem::size_of::<Uniforms>() as u64
                        ),
                    },
                    count: None,
                }],
            });
        let image_bind_group_layout =
            device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
                label: Some("Image Texture Bind Group Layout"),
                entries: &[
                    wgpu::BindGroupLayoutEntry {
                        binding: 0,
                        visibility: wgpu::ShaderStages::FRAGMENT,
                        ty: wgpu::BindingType::Texture {
                            multisampled: false,
                            view_dimension: wgpu::TextureViewDimension::D2,
                            sample_type: wgpu::TextureSampleType::Float { filterable: true },
                        },
                        count: None,
                    },
                    wgpu::BindGroupLayoutEntry {
                        binding: 1,
                        visibility: wgpu::ShaderStages::FRAGMENT,
                        ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
                        count: None,
                    },
                ],
            });
        let image_nearest_sampler =
            device.create_sampler(&image_sampler_descriptor(ImageSampling::Nearest));
        let image_linear_sampler =
            device.create_sampler(&image_sampler_descriptor(ImageSampling::Linear));
        let text_glyph_atlas = TextGlyphAtlas::new(
            &device,
            &image_bind_group_layout,
            GlyphSamplers {
                nearest: &image_nearest_sampler,
                linear: &image_linear_sampler,
            },
            TEXT_GLYPH_ATLAS_MIN_SIZE,
        );
        let viewport_uniforms = ViewportUniforms::default();

        static GLASS_MATERIAL_FOLDS: DebugToggle =
            DebugToggle::new("CRANPOSE_GLASS_MATERIAL_FOLDS");
        if GLASS_MATERIAL_FOLDS.equals("1") {
            cranpose_ui_graphics::set_glass_material_folds(true);
        } else if GLASS_MATERIAL_FOLDS.equals("0") {
            cranpose_ui_graphics::set_glass_material_folds(false);
        }
        log::info!(
            "[gpu-init] liquid glass material folds {}",
            if cranpose_ui_graphics::glass_material_folds_enabled() {
                "on: a pipeline per material's feature set"
            } else {
                "off: one pipeline per blend mode"
            }
        );

        #[cfg(not(target_arch = "wasm32"))]
        let crate::pipeline_disk_cache::Loaded {
            cache: pipeline_cache,
            records,
            updated,
            holds_recorded,
        } = crate::pipeline_disk_cache::load(&device);
        #[cfg(not(target_arch = "wasm32"))]
        let first_screen: Vec<u64> = records
            .shapes
            .iter()
            .filter(|shape| shape.first_screen)
            .map(|shape| shape.entry)
            .collect();
        #[cfg(target_arch = "wasm32")]
        let (pipeline_cache, first_screen): (Option<wgpu::PipelineCache>, Vec<u64>) =
            (None, Vec::new());
        let recorder = crate::pipeline_recorder::PipelineRecorder::default();
        #[cfg(not(target_arch = "wasm32"))]
        let pipeline_cache_watcher = crate::pipeline_disk_cache::spawn_persist_watcher(
            pipeline_cache.clone(),
            Arc::clone(&records),
            recorder.clone(),
        );

        let effects_started = Instant::now();
        let pipeline_compiler =
            PipelineCompiler::for_compilation(pipeline_compilation, adapter_backend);
        let depth_gate = crate::depth_gate::DepthGate::new(
            crate::draw_pass::lays_interiors_first(adapter_backend),
            pipeline_compiler.is_active(),
        );
        let effect_renderer = EffectRenderer::new(
            &device,
            pipeline_compiler.clone(),
            pipeline_cache.clone(),
            composition_format,
            adapter_backend,
            recorder.clone(),
        );
        let output_converter = OutputConverter::new(&device, adapter_backend, display_format);
        let screenshot_converter =
            OutputConverter::new(&device, adapter_backend, wgpu::TextureFormat::Rgba8Unorm);
        let effects_ms = instant_ms(effects_started, Instant::now());
        let mut frame_graph_executor = WgpuFrameGraphExecutor::new();
        frame_graph_executor.init_pass_timing(&device, &queue);
        #[cfg(target_arch = "wasm32")]
        frame_graph_executor.batch_queue_writes(adapter_backend == wgpu::Backend::BrowserWebGpu);
        let shape_shader = SharedShader::new(
            &device,
            adapter_backend,
            "Shape Shader",
            shape_shader_code(run_store.mode()),
            &[Some(&uniform_bind_group_layout), Some(run_store.layout())],
        );
        if run_store.mode().trig_fill {
            let fill = ArcTrigFill::new(
                &device,
                adapter_backend,
                pipeline_cache.clone(),
                shape_shader.clone(),
                recorder.clone(),
            );
            if first_screen.contains(&crate::arc_trig_fill::FIRST_SCREEN_KEY) {
                fill.warm(&pipeline_compiler);
            }
            run_store.attach_trig_fill(fill);
        }
        let shape_pipelines = ShapePipelines::new(
            ShapePipelineFactory {
                device: Arc::clone(&device),
                cache: pipeline_cache.clone(),
                format: composition_format,
                shader: shape_shader,
                mode: run_store.mode(),
            },
            adapter_backend,
            &pipeline_compiler,
            first_screen
                .into_iter()
                .filter(|&bits| bits != crate::arc_trig_fill::FIRST_SCREEN_KEY)
                .filter_map(ShapePipelineKey::from_bits),
            recorder.clone(),
        );
        let image_layouts = [
            Some(&uniform_bind_group_layout),
            Some(&image_bind_group_layout),
        ];
        let image_shader = SharedShader::new(
            &device,
            adapter_backend,
            "Image Shader",
            shaders::IMAGE,
            &image_layouts,
        );
        let glyph_atlas_shader = SharedShader::new(
            &device,
            adapter_backend,
            "Glyph Atlas Shader",
            shaders::GLYPH_ATLAS,
            &image_layouts,
        );
        let rrect_shadow_shader = SharedShader::new(
            &device,
            adapter_backend,
            "Round Rect Shadow Shader",
            shaders::RRECT_SHADOW,
            &[Some(&uniform_bind_group_layout)],
        );

        let mut renderer = Self {
            device,
            queue,
            device_errors,
            renderer_epoch,
            composition_format,
            #[cfg(not(target_arch = "wasm32"))]
            display_format,
            composition_target: None,
            output_converter,
            screenshot_converter,
            adapter_backend,
            pipeline_cache,
            #[cfg(not(target_arch = "wasm32"))]
            _pipeline_cache_watcher: pipeline_cache_watcher,
            #[cfg(not(target_arch = "wasm32"))]
            recent_after_update: updated.then(|| Arc::clone(&records)),
            recorder,
            #[cfg(not(target_arch = "wasm32"))]
            landing: pipeline_compiler.landing(),
            pipeline_compiler,
            #[cfg(not(target_arch = "wasm32"))]
            built_at_frame_start: 0,
            drew_placeholder: false,
            shape_pipelines,
            image_pipeline: [
                FixedPipeline::new("image/src-over"),
                FixedPipeline::new("image/src-over/depth"),
                FixedPipeline::new("image/src-over/alpha-mask"),
                FixedPipeline::new("image/src-over/alpha-mask/depth"),
                FixedPipeline::new("image/src-over/rounded"),
                FixedPipeline::new("image/src-over/rounded/depth"),
                FixedPipeline::new("image/src-over/alpha-mask/rounded"),
                FixedPipeline::new("image/src-over/alpha-mask/rounded/depth"),
            ],
            image_pipeline_dst_out: [
                FixedPipeline::new("image/dst-out"),
                FixedPipeline::new("image/dst-out/depth"),
                FixedPipeline::new("image/dst-out/alpha-mask"),
                FixedPipeline::new("image/dst-out/alpha-mask/depth"),
                FixedPipeline::new("image/dst-out/rounded"),
                FixedPipeline::new("image/dst-out/rounded/depth"),
                FixedPipeline::new("image/dst-out/alpha-mask/rounded"),
                FixedPipeline::new("image/dst-out/alpha-mask/rounded/depth"),
            ],
            glyph_atlas_pipeline: [
                FixedPipeline::new("glyph/atlas"),
                FixedPipeline::new("glyph/atlas/depth"),
                FixedPipeline::new("glyph/atlas/aligned"),
                FixedPipeline::new("glyph/atlas/aligned/depth"),
                FixedPipeline::new("glyph/atlas/turned"),
                FixedPipeline::new("glyph/atlas/turned/depth"),
            ],
            rrect_shadow_pipeline: [
                FixedPipeline::new("shadow/rrect"),
                FixedPipeline::new("shadow/rrect/depth"),
            ],
            image_shader,
            glyph_atlas_shader,
            rrect_shadow_shader,
            depth_targets: crate::idle_pool::IdlePool::default(),
            depth_gate,
            surface_atlas_sizes: crate::frame::SurfaceAtlasSizes::for_backend(adapter_backend),
            uniform_bind_group_layout,
            image_bind_group_layout,
            image_nearest_sampler,
            image_linear_sampler,
            text_fonts,
            viewport_uniforms,
            run_store,
            image_texture_cache: BoundedLruCache::with_capacity_at_least_one(
                MAX_TEXTURE_CACHE_ITEMS,
            ),
            image_texture_cache_bytes: 0,
            external_texture_bindings: BoundedLruCache::with_capacity_at_least_one(
                MAX_EXTERNAL_TEXTURE_BINDINGS,
            ),
            text_image_cache: BoundedLruCache::with_capacity_at_least_one(
                MAX_TEXT_IMAGE_CACHE_ITEMS,
            ),
            text_glyph_atlas,
            text_glyph_run_cache: BoundedLruCache::with_capacity_at_least_one(
                MIN_TEXT_GLYPH_RUN_CACHE_ITEMS,
            ),
            text_glyph_runs_drawn: 0,
            text_glyph_gpu_run_cache: BoundedLruCache::with_capacity_at_least_one(
                MAX_TEXT_GLYPH_GPU_RUN_CACHE_ITEMS,
            ),
            text_glyph_run_arena: GlyphRunArena::new(copy_free),
            copy_free,
            text_glyph_run_frame: 0,
            text_glyph_mask_cache: SoftwareGlyphRasterCache::with_capacity_at_least_one(
                MAX_TEXT_GLYPH_MASK_CACHE_ITEMS,
            ),
            text_line_index_cache: TextLineIndexCache::new(MAX_TEXT_LINE_INDEX_CACHE_ITEMS),
            scratch_image_vertices: Vec::new(),
            scratch_image_indices: Vec::new(),
            scratch_glyph_instances: GlyphInstances::default(),
            scratch_image_cmds: Vec::new(),
            scratch_image_clips: Vec::new(),
            scratch_glyph_cmds: Vec::new(),
            scratch_glyph_moved: Vec::new(),
            scratch_shadow_instances: Vec::new(),
            scratch_arena_draws: Vec::new(),
            scratch_pipelines: PipelineDemand::default(),
            scratch_held: crate::draw_pass::HeldDraws::default(),
            scratch_text_glyph_run: Vec::new(),
            scratch_text_glyph_entries: Vec::new(),
            run_glyph_scratch: RunGlyphScratch::default(),
            frame_graph_executor,
            stage_side_scratch: StageSideScratch::default(),
            deferred_offscreen_releases: Vec::new(),
            effect_renderer,
            layer_cache: LayerCache::new(),
            ablation: Ablation::default(),
            ablation_frames: 0,
            nesting_overflow_reported: false,
            backdrop_gates: HashMap::default(),
            fill_gates: HashMap::default(),
            effect_gates: HashMap::default(),
            source_gates: HashMap::default(),
            transparent_sources: HashMap::default(),
            shadow_surface_cache: BoundedLruCache::with_capacity_at_least_one(
                MAX_SHADOW_SURFACE_CACHE_ITEMS,
            ),
            shadow_surface_cache_bytes: 0,
            frame_stats: gpu_stats::FrameStats::default(),
            last_frame_stats: None,
            pending_frame_warmup_frames: 0,
            frame_count: 0,
            shader_warm_ups_queued: 0,
        };
        #[cfg(not(target_arch = "wasm32"))]
        renderer.warm_first_screen(&records, holds_recorded);
        renderer.warm_requested_shaders();
        log::info!(
            "[gpu-init] {:?} renderer ready in {:.1} ms (effects {:.1} ms)",
            adapter_backend,
            instant_ms(construction_started, Instant::now()),
            effects_ms,
        );
        renderer
    }

    /// Prepares the first screen the last launches recorded. The first frame
    /// waits for its shapes and fixed pipelines, but draws placeholders for
    /// runtime shaders that have not compiled: those queue behind, since a
    /// Mali driver compiles largely one pipeline at a time. A driver cache
    /// that `holds_recorded` pipelines builds them as cache hits, so those
    /// draw from it, and need no stand-in.
    #[cfg(not(target_arch = "wasm32"))]
    fn warm_first_screen(
        &mut self,
        records: &crate::pipeline_records::PipelineRecords,
        holds_recorded: bool,
    ) {
        self.warm_fixed_pipelines(&|label| {
            records
                .fixed
                .iter()
                .any(|fixed| fixed.first_screen && fixed.entry == label)
        });
        if holds_recorded {
            self.effect_renderer.trust_cached(records);
        }
        self.effect_renderer.warm_first_screen_shaders(
            records
                .shaders
                .iter()
                .filter(|shader| shader.first_screen)
                .map(|shader| &shader.entry),
        );
        if !records.shapes.iter().any(|shape| shape.first_screen) {
            self.warm_every_screen();
        }
    }

    /// Queues the pipelines every screen draws with, for a launch with no
    /// record of its first screen: flat solid rects and text on whole
    /// pixels. They build while the first screen composes, when the
    /// compiler's threads have nothing else; on a Mate 20 X the gauntlet's
    /// first content showed 333-366 ms after the first frame with them
    /// against 383-399 ms without.
    #[cfg(not(target_arch = "wasm32"))]
    fn warm_every_screen(&mut self) {
        self.shape_pipelines.warm(ShapePipelineKey::SOLID_RECT);
        self.glyph_atlas_pipeline_resource(false, GlyphQuads::Aligned)
            .prefetch(
                CompileLane::WarmUp,
                &self.pipeline_compiler,
                self.adapter_backend,
                || self.glyph_atlas_pipeline_job(false, GlyphQuads::Aligned),
            );
    }

    fn ensure_shape_pipeline(&mut self, key: ShapePipelineKey, vertices: u64) {
        self.shape_pipelines.ensure(key, vertices);
    }

    /// Queues the shader warm-ups requested since the last call, each at the
    /// target it draws to (`cranpose_ui_graphics::request_shader_warm_ups`).
    fn warm_requested_shaders(&mut self) {
        let requested = cranpose_ui_graphics::shader_warm_ups_after(self.shader_warm_ups_queued);
        if requested.is_empty() {
            return;
        }
        self.shader_warm_ups_queued += requested.len();
        self.effect_renderer.warm_shaders(&requested);
    }

    /// Queues every pipeline recent launches of another build drew with:
    /// after an update their compiled pipelines are gone, and each would
    /// otherwise compile inside the first frame that draws with it.
    #[cfg(not(target_arch = "wasm32"))]
    fn warm_recent(&mut self, records: &crate::pipeline_records::PipelineRecords) {
        self.warm_fixed_pipelines(&|label| records.fixed.iter().any(|fixed| fixed.entry == label));
        let shapes = records
            .shapes
            .iter()
            .map(|shape| shape.entry)
            .filter(|&bits| bits != crate::arc_trig_fill::FIRST_SCREEN_KEY)
            .filter_map(ShapePipelineKey::from_bits);
        for key in shapes {
            self.shape_pipelines.warm(key);
        }
        self.effect_renderer
            .warm_recorded_shaders(records.shaders.iter().map(|shader| &shader.entry));
    }

    /// Queues the fixed pipelines whose labels `wanted` accepts.
    #[cfg(not(target_arch = "wasm32"))]
    fn warm_fixed_pipelines(&self, wanted: &dyn Fn(&str) -> bool) {
        let compiler = self.effect_renderer.compiler();
        self.for_each_fixed_pipeline(&mut |pipeline, job| {
            if wanted(pipeline.label()) {
                pipeline.queue(
                    compiler,
                    crate::pipeline_compiler::CompileLane::WarmUp,
                    self.adapter_backend,
                    job(),
                );
            }
        });
    }

    /// Builds here, ahead of the first frame, the pipelines named by the
    /// first screens in `records` that this renderer's backend drew (see
    /// [`crate::WgpuRenderer::build_first_screen_pipelines`]).
    pub(crate) fn build_first_screen(&mut self, records: &[u8]) {
        let records = crate::pipeline_records::decode_first_screens(records, self.adapter_backend);
        for shape in &records.shapes {
            if let Some(key) = ShapePipelineKey::from_bits(shape.entry) {
                self.shape_pipelines.build_ahead(key);
            }
        }
        self.for_each_fixed_pipeline(&mut |pipeline, job| {
            if records
                .fixed
                .iter()
                .any(|fixed| fixed.entry == pipeline.label())
            {
                pipeline.build_ahead(&self.recorder, self.adapter_backend, job());
            }
        });
    }

    /// See [`crate::WgpuRenderer::first_screen_pipelines`].
    #[cold]
    #[inline(never)]
    pub(crate) fn first_screen_pipelines(&self) -> Option<Vec<u8>> {
        self.recorder.first_screen_records(self.adapter_backend)
    }

    /// Hands `visit` every fixed pipeline, the effect renderer's too, with
    /// the job that builds it.
    fn for_each_fixed_pipeline(
        &self,
        visit: &mut dyn FnMut(&FixedPipeline, &dyn Fn() -> FixedPipelineJob),
    ) {
        self.effect_renderer.for_each_fixed(&self.device, visit);
        for depth in [false, true] {
            for (alpha_mask, rounded) in
                [(false, false), (true, false), (false, true), (true, true)]
            {
                let variant = ImageVariant {
                    depth,
                    alpha_mask,
                    rounded,
                };
                for blend_mode in [BlendMode::SrcOver, BlendMode::DstOut] {
                    visit(self.image_pipeline_resource(blend_mode, variant), &|| {
                        Box::new(self.image_pipeline_job(blend_mode, variant))
                    });
                }
            }
            for quads in [GlyphQuads::Plain, GlyphQuads::Aligned, GlyphQuads::Turned] {
                visit(self.glyph_atlas_pipeline_resource(depth, quads), &|| {
                    Box::new(self.glyph_atlas_pipeline_job(depth, quads))
                });
            }
            visit(&self.rrect_shadow_pipeline[usize::from(depth)], &|| {
                Box::new(self.rrect_shadow_pipeline_job(depth))
            });
        }
    }

    fn image_pipeline_resource(
        &self,
        blend_mode: BlendMode,
        variant: ImageVariant,
    ) -> &FixedPipeline {
        let pipelines = match blend_mode {
            BlendMode::DstOut => &self.image_pipeline_dst_out,
            _ => &self.image_pipeline,
        };
        &pipelines[usize::from(variant.depth)
            + 2 * usize::from(variant.alpha_mask)
            + 4 * usize::from(variant.rounded)]
    }

    fn image_pipeline_job(
        &self,
        blend_mode: BlendMode,
        variant: ImageVariant,
    ) -> impl FnOnce() -> wgpu::RenderPipeline + CompilerSend + 'static {
        let device = Arc::clone(&self.device);
        let cache = self.pipeline_cache.clone();
        let format = self.composition_format;
        let shader = self.image_shader.clone();
        move || {
            create_image_pipeline(
                &device,
                cache.as_ref(),
                format,
                &shader,
                blend_mode,
                variant,
            )
        }
    }

    /// Queues the builds of the image pipelines `cmds` draw with that are
    /// missing on `lane` (see [`FixedPipeline::prefetch`]), handing
    /// `awaited` each still to land.
    pub(crate) fn prefetch_image_pipelines(
        &self,
        cmds: &[ImageDrawCmd],
        blend_mode: BlendMode,
        (depth, lane): (bool, CompileLane),
        awaited: &mut impl FnMut(Pending<wgpu::RenderPipeline>),
    ) {
        let mut last = None;
        for cmd in cmds {
            let variant = ImageVariant::of(cmd, depth);
            if last.replace(variant) == Some(variant) {
                continue;
            }
            if let Some(pending) = self.image_pipeline_resource(blend_mode, variant).prefetch(
                lane,
                &self.pipeline_compiler,
                self.adapter_backend,
                || self.image_pipeline_job(blend_mode, variant),
            ) {
                awaited(pending);
            }
        }
    }

    /// Queues the builds of the glyph pipelines `cmds` draw with that are
    /// missing on `lane` (see [`FixedPipeline::prefetch`]), handing
    /// `awaited` each still to land.
    pub(crate) fn prefetch_glyph_pipelines(
        &self,
        cmds: &[GlyphDrawCmd],
        (depth, lane): (bool, CompileLane),
        awaited: &mut impl FnMut(Pending<wgpu::RenderPipeline>),
    ) {
        let mut last = None;
        for cmd in cmds {
            let quads = cmd.quads();
            if last.replace(quads) == Some(quads) {
                continue;
            }
            if let Some(pending) = self.glyph_atlas_pipeline_resource(depth, quads).prefetch(
                lane,
                &self.pipeline_compiler,
                self.adapter_backend,
                || self.glyph_atlas_pipeline_job(depth, quads),
            ) {
                awaited(pending);
            }
        }
    }

    /// Queues the build of the shadow pipeline on `lane` when it is missing
    /// (see [`FixedPipeline::prefetch`]), handing `awaited` the pipeline
    /// while it is still to land.
    pub(crate) fn prefetch_shadow_pipeline(
        &self,
        (depth, lane): (bool, CompileLane),
        awaited: &mut impl FnMut(Pending<wgpu::RenderPipeline>),
    ) {
        if let Some(pending) = self.rrect_shadow_pipeline[usize::from(depth)].prefetch(
            lane,
            &self.pipeline_compiler,
            self.adapter_backend,
            || self.rrect_shadow_pipeline_job(depth),
        ) {
            awaited(pending);
        }
    }

    pub(crate) fn image_pipeline(
        &self,
        blend_mode: BlendMode,
        variant: ImageVariant,
    ) -> &wgpu::RenderPipeline {
        self.image_pipeline_resource(blend_mode, variant).for_draw(
            &self.recorder,
            self.adapter_backend,
            || self.image_pipeline_job(blend_mode, variant)(),
        )
    }

    fn glyph_atlas_pipeline_job(
        &self,
        depth: bool,
        quads: GlyphQuads,
    ) -> impl FnOnce() -> wgpu::RenderPipeline + CompilerSend + 'static {
        let device = Arc::clone(&self.device);
        let cache = self.pipeline_cache.clone();
        let format = self.composition_format;
        let shader = self.glyph_atlas_shader.clone();
        move || {
            create_glyph_atlas_pipeline(&device, cache.as_ref(), format, &shader, (depth, quads))
        }
    }

    /// The glyph pipeline for a pass with a depth buffer or not, drawing
    /// `quads`.
    fn glyph_atlas_pipeline(&self, depth: bool, quads: GlyphQuads) -> &wgpu::RenderPipeline {
        self.glyph_atlas_pipeline_resource(depth, quads).for_draw(
            &self.recorder,
            self.adapter_backend,
            || self.glyph_atlas_pipeline_job(depth, quads)(),
        )
    }

    fn rrect_shadow_pipeline_job(
        &self,
        depth: bool,
    ) -> impl FnOnce() -> wgpu::RenderPipeline + CompilerSend + 'static {
        let device = Arc::clone(&self.device);
        let cache = self.pipeline_cache.clone();
        let format = self.composition_format;
        let shader = self.rrect_shadow_shader.clone();
        move || create_rrect_shadow_pipeline(&device, cache.as_ref(), format, &shader, depth)
    }

    /// Draws `instances` of the pass's uploaded shadow instances, bound with
    /// `uniform_slot` under `scissor`.
    pub(crate) fn draw_shadow_instances(
        &self,
        pass: &mut TrackedPass<'_, '_>,
        buffer: &BufferUpload,
        uniform_slot: usize,
        instances: std::ops::Range<u32>,
        scissor: Option<(u32, u32, u32, u32)>,
        frame: PassFrame,
    ) -> Result<(), String> {
        let depth = frame.depth;
        let pipeline = self.rrect_shadow_pipeline[usize::from(depth)].for_draw(
            &self.recorder,
            self.adapter_backend,
            || self.rrect_shadow_pipeline_job(depth)(),
        );
        let (x, y, width, height) = frame.scissor(scissor);
        pass.set_scissor_rect(x, y, width, height);
        pass.set_pipeline(pipeline);
        self.viewport_uniforms.bind(pass, uniform_slot)?;
        pass.set_vertex_buffer(0, buffer.slice());
        pass.draw(0..SHADOW_QUAD_CORNERS, instances);
        self.frame_stats.add_draw_calls(1);
        Ok(())
    }

    pub(crate) fn upload_shadow_instances<C: FrameCommandRecorder>(
        &self,
        recorder: &mut C,
        instances: &[ShadowInstance],
    ) -> BufferUpload {
        recorder.upload_buffer(
            UploadAllocatorSpec::vertex(
                "Shadow Instance Buffer",
                std::mem::size_of::<ShadowInstance>() as u64,
            ),
            &self.device,
            bytemuck::cast_slice(instances),
        )
    }

    fn glyph_atlas_pipeline_resource(&self, depth: bool, quads: GlyphQuads) -> &FixedPipeline {
        let kind = match quads {
            GlyphQuads::Plain => 0,
            GlyphQuads::Aligned => 1,
            GlyphQuads::Turned => 2,
        };
        &self.glyph_atlas_pipeline[usize::from(depth) + 2 * kind]
    }

    /// The transient depth buffer for a target of `size`, created on first
    /// use; the few sizes recent frames used are kept.
    pub(crate) fn depth_target(&mut self, size: (u32, u32)) -> wgpu::TextureView {
        const KEPT_DEPTH_TARGETS: usize = 4;
        let view = match self.depth_targets.take(|(kept, _)| *kept == size) {
            Some((_, view)) => view,
            None => self.create_depth_target(size),
        };
        self.depth_targets
            .put((size, view.clone()), KEPT_DEPTH_TARGETS, u64::MAX, |_| 0);
        view
    }

    fn create_depth_target(&self, size: (u32, u32)) -> wgpu::TextureView {
        let texture = self.device.create_texture(&wgpu::TextureDescriptor {
            label: Some("Opaque interior depth"),
            size: wgpu::Extent3d {
                width: size.0,
                height: size.1,
                depth_or_array_layers: 1,
            },
            mip_level_count: 1,
            sample_count: 1,
            dimension: wgpu::TextureDimension::D2,
            format: DEPTH_FORMAT,
            usage: wgpu::TextureUsages::RENDER_ATTACHMENT
                | wgpu::TextureUsages::TRANSIENT_ATTACHMENT,
            view_formats: &[],
        });
        texture.create_view(&wgpu::TextureViewDescriptor::default())
    }

    /// The binding a draw of `image` sampled by `sampling` takes, uploading
    /// the image the first time a draw needs it.
    fn image_binding(
        &mut self,
        image: &ImageBitmap,
        sampling: ImageSampling,
    ) -> Result<ImageBinding, String> {
        if image.format() == cranpose_ui_graphics::ImagePixelFormat::External {
            return self.external_texture_binding(image, sampling);
        }
        if let Some(cached) = self.image_texture_cache.get(&image.id()) {
            return Ok(cached.bind_groups.binding(sampling));
        }

        let size = wgpu::Extent3d {
            width: image.width(),
            height: image.height(),
            depth_or_array_layers: 1,
        };

        let alpha_mask = matches!(
            image.format(),
            cranpose_ui_graphics::ImagePixelFormat::Alpha8 { .. }
        );
        let (format, bytes_per_pixel) = if alpha_mask {
            (wgpu::TextureFormat::R8Unorm, 1)
        } else {
            (wgpu::TextureFormat::Rgba8Unorm, 4)
        };

        let texture = self.device.create_texture(&wgpu::TextureDescriptor {
            label: Some("Image Texture"),
            size,
            mip_level_count: 1,
            sample_count: 1,
            dimension: wgpu::TextureDimension::D2,
            format,
            usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
            view_formats: &[],
        });

        let upload_stats = self.frame_graph_executor.upload_texture(
            &self.queue,
            wgpu::TexelCopyTextureInfo {
                texture: &texture,
                mip_level: 0,
                origin: wgpu::Origin3d::ZERO,
                aspect: wgpu::TextureAspect::All,
            },
            image.pixels(),
            wgpu::TexelCopyBufferLayout {
                offset: 0,
                bytes_per_row: Some(bytes_per_pixel * image.width()),
                rows_per_image: Some(image.height()),
            },
            size,
        );
        self.frame_stats.record_command_stats(upload_stats);

        let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
        let bytes = image.pixels().len();
        let cached = CachedImageTexture {
            bind_groups: self.image_bind_groups(&view, alpha_mask),
            _texture: texture,
            _view: view,
            bytes,
        };
        let binding = cached.bind_groups.binding(sampling);
        if let Some(replaced) = self.image_texture_cache.put(image.id(), cached) {
            self.image_texture_cache_bytes = self
                .image_texture_cache_bytes
                .saturating_sub(replaced.bytes);
        }
        self.image_texture_cache_bytes += bytes;
        while self.image_texture_cache_bytes > MAX_IMAGE_TEXTURE_CACHE_BYTES
            && self.image_texture_cache.len() > 1
        {
            let Some((_, evicted)) = self.image_texture_cache.pop_lru() else {
                break;
            };
            self.image_texture_cache_bytes =
                self.image_texture_cache_bytes.saturating_sub(evicted.bytes);
        }
        Ok(binding)
    }

    fn external_texture_binding(
        &mut self,
        image: &ImageBitmap,
        sampling: ImageSampling,
    ) -> Result<ImageBinding, String> {
        let key = image.storage_id();
        if let Some(bind_groups) = self.external_texture_bindings.get(&key) {
            return Ok(bind_groups.binding(sampling));
        }
        let view = image
            .external_handle::<wgpu::TextureView>()
            .ok_or("an external texture's handle is not a wgpu::TextureView")?;
        let bind_groups = self.image_bind_groups(view, false);
        let binding = bind_groups.binding(sampling);
        self.external_texture_bindings.put(key, bind_groups);
        Ok(binding)
    }

    fn image_bind_groups(&self, view: &wgpu::TextureView, alpha_mask: bool) -> ImageBindGroups {
        ImageBindGroups {
            alpha_mask,
            nearest: self.image_bind_group(view, &self.image_nearest_sampler),
            linear: self.image_bind_group(view, &self.image_linear_sampler),
        }
    }

    fn image_bind_group(
        &self,
        view: &wgpu::TextureView,
        sampler: &wgpu::Sampler,
    ) -> wgpu::BindGroup {
        self.device.create_bind_group(&wgpu::BindGroupDescriptor {
            label: Some("Image Texture Bind Group"),
            layout: &self.image_bind_group_layout,
            entries: &[
                wgpu::BindGroupEntry {
                    binding: 0,
                    resource: wgpu::BindingResource::TextureView(view),
                },
                wgpu::BindGroupEntry {
                    binding: 1,
                    resource: wgpu::BindingResource::Sampler(sampler),
                },
            ],
        })
    }

    pub(crate) fn max_texture_dim(&self) -> u32 {
        self.effect_renderer.max_texture_dim()
    }

    /// A pooled texture that outlives the frame: layer cache entries and
    /// cached shadow surfaces.
    pub(crate) fn acquire_retained_surface(&mut self, width: u32, height: u32) -> OffscreenTarget {
        self.effect_renderer
            .acquire_offscreen(&self.device, width, height, Some(&self.frame_stats))
    }

    fn frame_root(
        &mut self,
        output_mode: OutputMode,
        output_view: Option<&wgpu::TextureView>,
        output_texture: Option<&wgpu::Texture>,
        viewport: (u32, u32),
        reads_page: bool,
    ) -> FrameRoot {
        if let (OutputMode::Display, Some(view), Some(texture)) =
            (output_mode, output_view, output_texture)
            && surface_is_direct_root(texture, self.composition_format, viewport, reads_page)
        {
            return FrameRoot::Surface(Rc::new(OffscreenTarget::from_surface(
                texture.clone(),
                view.clone(),
            )));
        }
        FrameRoot::Composition(self.take_composition_target(viewport.0.max(1), viewport.1.max(1)))
    }

    fn take_composition_target(&mut self, width: u32, height: u32) -> CompositionTarget {
        if let Some(target) = self.composition_target.take()
            && target.target.width == width
            && target.target.height == height
        {
            return target;
        }
        let target = Rc::new(OffscreenTarget::new(
            &self.device,
            self.composition_format,
            width,
            height,
        ));
        let output_bind_group = self.output_converter.bind_group(&self.device, &target.view);
        CompositionTarget {
            target,
            output_bind_group,
        }
    }

    fn transient_offscreen_descriptor(
        &self,
        label: &'static str,
        width: u32,
        height: u32,
    ) -> FrameTextureDescriptor {
        let max_texture_dim = self.max_texture_dim();
        FrameTextureDescriptor::render_attachment(
            label,
            width.min(max_texture_dim),
            height.min(max_texture_dim),
            self.composition_format,
        )
    }

    /// A texture of the given size that stays transparent: the input of a
    /// runtime shader whose layer draws nothing itself, so the shader needs
    /// no surface pass and reads the same empty content every frame.
    #[inline(never)]
    pub(crate) fn transparent_source<C: FrameCommandRecorder>(
        &mut self,
        recorder: &mut C,
        width: u32,
        height: u32,
    ) -> Rc<OffscreenTarget> {
        if let Some(source) = self.transparent_sources.get(&(width, height)) {
            return Rc::clone(source);
        }
        if self.transparent_sources.len() >= MAX_TRANSPARENT_SOURCES {
            for (_, source) in self.transparent_sources.drain() {
                if let Ok(target) = Rc::try_unwrap(source) {
                    self.deferred_offscreen_releases.push(target);
                }
            }
        }
        let source = Rc::new(self.acquire_retained_surface(width, height));
        self.clear_target(
            recorder,
            &source.view,
            wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
        );
        self.transparent_sources
            .insert((width, height), Rc::clone(&source));
        source
    }

    fn defer_offscreen_release(&mut self, target: OffscreenTarget) {
        self.deferred_offscreen_releases.push(target);
    }

    fn flush_deferred_offscreen_releases(&mut self) {
        let layer_cache = &mut self.layer_cache;
        let mut retire = |gate: &mut AdmissionGate| {
            let seen = gate.end_frame();
            if !seen && let Some(dead) = gate.dead_entry() {
                layer_cache.remove(&dead);
            }
            seen
        };
        self.backdrop_gates.retain(|_, gate| retire(gate));
        self.fill_gates.retain(|_, gate| retire(gate));
        self.effect_gates.retain(|_, gate| retire(gate));
        self.source_gates.retain(|_, gate| retire(gate));
        for target in self.deferred_offscreen_releases.drain(..) {
            self.effect_renderer.release_offscreen(target);
        }
        self.layer_cache.end_frame();
        for (transient, target) in self.layer_cache.take_released() {
            match transient {
                Some(descriptor) => self
                    .frame_graph_executor
                    .return_cached_transient(descriptor, target),
                None => self.effect_renderer.release_offscreen(target),
            }
        }
        self.effect_renderer.end_offscreen_frame();
        self.frame_graph_executor.end_transient_frame();
        self.depth_targets.end_frame();
        self.surface_atlas_sizes.end_frame();
    }

    fn insert_cached_shadow_surface(
        &mut self,
        key: ShadowSurfaceCacheKey,
        target: Rc<OffscreenTarget>,
    ) {
        let byte_size = offscreen_byte_size(target.width, target.height);
        while self.shadow_surface_cache_bytes + byte_size > MAX_SHADOW_SURFACE_CACHE_BYTES {
            let Some((_, evicted)) = self.shadow_surface_cache.pop_lru() else {
                break;
            };
            self.shadow_surface_cache_bytes = self
                .shadow_surface_cache_bytes
                .saturating_sub(evicted.byte_size);
        }
        let cached = CachedShadowSurface { target, byte_size };
        if let Some((_, replaced)) = self.shadow_surface_cache.push(key, cached) {
            self.shadow_surface_cache_bytes = self
                .shadow_surface_cache_bytes
                .saturating_sub(replaced.byte_size);
        }
        self.shadow_surface_cache_bytes = self.shadow_surface_cache_bytes.saturating_add(byte_size);
    }
}
fn frame_stats_need_warmup_frame(snapshot: &gpu_stats::FrameStatsSnapshot) -> bool {
    snapshot.layer_cache_misses > 0
        || snapshot.shadow_shape_cache_misses > 0
        || snapshot.text_image_cache_misses > 0
        || snapshot.text_glyph_atlas_misses > 0
}

fn update_frame_warmup_budget(pending_frames: &mut u8, snapshot: &gpu_stats::FrameStatsSnapshot) {
    if *pending_frames > 0 {
        *pending_frames = pending_frames.saturating_sub(1);
    } else if frame_stats_need_warmup_frame(snapshot) {
        *pending_frames = CACHE_MISS_WARMUP_FRAMES;
    }
}

impl GpuRenderer {
    #[expect(clippy::too_many_arguments)]
    pub fn render(
        &mut self,
        texture: &wgpu::Texture,
        view: &wgpu::TextureView,
        width: u32,
        height: u32,
        packet: FramePacket,
        surface_epoch: u64,
        returns: &mut RenderReturns,
    ) -> Result<(), String> {
        self.render_internal(
            width,
            height,
            packet,
            surface_epoch,
            returns,
            OutputMode::Display,
            Some(view),
            Some(texture),
        )
    }

    #[expect(clippy::too_many_arguments)]
    fn render_internal(
        &mut self,
        width: u32,
        height: u32,
        packet: FramePacket,
        surface_epoch: u64,
        returns: &mut RenderReturns,
        output_mode: OutputMode,
        output_view: Option<&wgpu::TextureView>,
        output_texture: Option<&wgpu::Texture>,
    ) -> Result<(), String> {
        let cancel_reason = if packet.renderer_epoch != self.renderer_epoch {
            Some(CancelReason::RendererEpoch)
        } else if packet.surface_epoch != surface_epoch {
            Some(CancelReason::SurfaceEpoch)
        } else if packet.viewport != (width, height) {
            Some(CancelReason::Viewport)
        } else {
            None
        };
        if let Some(reason) = cancel_reason {
            return Self::cancel_packet(packet, reason, returns);
        }
        if self.device_errors.take_poison() {
            return Self::cancel_packet(packet, CancelReason::DeviceError, returns);
        }
        returns.frame_id = packet.frame_id;
        let render_start = Instant::now();
        #[cfg(not(target_arch = "wasm32"))]
        let builds_before = thread_builds();
        self.warm_requested_shaders();
        self.shape_pipelines.begin_frame();
        self.recorder.begin_frame();
        self.begin_placeholder_frame();
        self.viewport_uniforms.begin_frame();
        self.run_store.begin_frame(gpu_stats_enabled());
        self.begin_text_glyph_run_frame();

        let text_cache_len = packet.text_cache_len;
        let reads_page = packet.root.contains_backdrop()
            || packet
                .overlay
                .as_ref()
                .is_some_and(LayerScene::contains_backdrop);
        let frame_root = self.frame_root(
            output_mode,
            output_view,
            output_texture,
            (width, height),
            reads_page,
        );
        let root = frame_root.target();
        let screenshot_bind_group = output_view.and_then(|_| {
            matches!(output_mode, OutputMode::Screenshot).then(|| {
                self.screenshot_converter
                    .bind_group(&self.device, &root.view)
            })
        });
        let output = frame_root.output(output_view, screenshot_bind_group.as_ref());
        let result = self.render_graph(root, packet, returns, output_mode, output);
        #[cfg(not(target_arch = "wasm32"))]
        self.effect_renderer
            .compiler()
            .spread_demand(thread_builds() == builds_before);
        if let FrameRoot::Composition(composition) = frame_root {
            self.composition_target = Some(composition);
        }
        let after_graph = Instant::now();
        self.flush_deferred_offscreen_releases();
        self.scratch_glyph_instances.end_frame();

        self.frame_stats
            .layer_cache_size
            .set(self.layer_cache.len() as u32);
        self.frame_stats
            .layer_cache_bytes
            .set(self.layer_cache.bytes());
        self.frame_stats.offscreen_pool_size.set(
            self.effect_renderer
                .retained_offscreen_count()
                .saturating_add(self.frame_graph_executor.retained_texture_count())
                .saturating_add(usize::from(self.composition_target.is_some())) as u32,
        );
        self.frame_stats.offscreen_pool_bytes.set(
            (self.effect_renderer.retained_offscreen_bytes() as u64)
                .saturating_add(self.frame_graph_executor.retained_texture_bytes())
                .saturating_add(self.composition_target.as_ref().map_or(0, |target| {
                    u64::from(target.target.width)
                        .saturating_mul(u64::from(target.target.height))
                        .saturating_mul(COMPOSITION_BYTES_PER_PIXEL)
                })),
        );
        self.frame_stats
            .text_pool_size
            .set(self.text_image_cache.len() as u32);
        self.frame_stats
            .image_cache_size
            .set(self.image_texture_cache.len() as u32);
        self.frame_stats.text_cache_size.set(text_cache_len as u32);
        self.effect_renderer
            .merge_and_reset_debug_counters(&self.frame_stats);
        self.frame_graph_executor.reset_upload_allocators();
        let snapshot = self.frame_stats.snapshot();
        if crate::frame_graph::frame_graph_pass_telemetry_threshold_ms().is_some() {
            log::warn!(
                "[wgpu-render-stage:frame-stats] layer_hit={} layer_miss={} miss_px={} \
                 offscreen_acq={} offscreen_new={} isolated={} draws={}",
                snapshot.layer_cache_hits,
                snapshot.layer_cache_misses,
                snapshot.layer_cache_miss_pixels,
                snapshot.offscreen_acquires,
                snapshot.offscreen_news,
                snapshot.isolated_layer_renders,
                snapshot.draw_calls,
            );
        }
        self.last_frame_stats = Some(snapshot);
        PRESENTED_FRAMES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
        update_frame_warmup_budget(&mut self.pending_frame_warmup_frames, &snapshot);
        let gpu_stats_on = gpu_stats_enabled();
        self.frame_stats
            .maybe_print_snapshot(snapshot, &mut self.frame_count, gpu_stats_on);
        if gpu_stats_on && self.frame_count.is_multiple_of(60) {
            gpu_stats::print_gpu_memory_report(&self.device, self.frame_count);
        }
        self.frame_graph_executor
            .end_pass_timing_frame(&self.device, &self.queue);
        self.frame_stats.reset();
        let after_stats = Instant::now();
        if let Some(total_ms) = should_log_wgpu_render_stage(render_start, after_stats) {
            log::warn!(
                "[wgpu-render-stage:render] total_ms={total_ms:.2} graph_ms={:.2} cleanup_stats_ms={:.2}",
                instant_ms(render_start, after_graph),
                instant_ms(after_graph, after_stats),
            );
        }
        if result.is_ok() {
            returns.outcome = PresentOutcome::Presented;
        }
        result
    }

    /// Returns a packet unrendered, handing its scene back for recycling.
    pub(crate) fn cancel_packet(
        packet: FramePacket,
        reason: CancelReason,
        returns: &mut RenderReturns,
    ) -> Result<(), String> {
        returns.scene = Some(FrameSceneStorage {
            root: packet.root,
            overlay: packet.overlay,
        });
        returns.frame_id = packet.frame_id;
        returns.outcome = PresentOutcome::Cancelled(reason);
        Ok(())
    }

    pub fn last_frame_stats(&self) -> Option<gpu_stats::FrameStatsSnapshot> {
        self.last_frame_stats
    }

    pub fn gpu_pass_timings(&self) -> crate::pass_timing::GpuPassTimingReport {
        self.frame_graph_executor.pass_timing_report()
    }

    pub fn needs_frame_warmup(&self) -> bool {
        self.pending_frame_warmup_frames > 0 || self.placeholder_awaits_frame()
    }

    #[cfg(target_arch = "wasm32")]
    pub(crate) fn pipeline_compiler(&self) -> &PipelineCompiler {
        &self.pipeline_compiler
    }

    /// Whether a frame must follow to replace a placeholder: a pipeline it
    /// waited for landed, or, with nothing to wake the app when one does,
    /// this frame drew one.
    fn placeholder_awaits_frame(&self) -> bool {
        #[cfg(not(target_arch = "wasm32"))]
        if let Some(landing) = &self.landing {
            return landing.landed() || (self.drew_placeholder && !landing.has_wake());
        }
        self.drew_placeholder
    }

    /// Starts a frame: once a pipeline a placeholder waited for has landed,
    /// the retained layers, which may hold that placeholder, are dropped.
    fn begin_placeholder_frame(&mut self) {
        self.drew_placeholder = false;
        #[cfg(not(target_arch = "wasm32"))]
        if let Some(landing) = &self.landing {
            self.built_at_frame_start = landing.built();
            if landing.take_landed() {
                self.layer_cache.clear();
            }
        }
    }

    /// Notes an effect drawn as its placeholder, or left out, while its
    /// pipelines compile, and waits for the next one to land.
    pub(crate) fn note_placeholder(&mut self) {
        self.drew_placeholder = true;
        self.recorder.hold_first_screen();
        self.frame_stats
            .placeholder_draws
            .set(self.frame_stats.placeholder_draws.get() + 1);
        #[cfg(not(target_arch = "wasm32"))]
        if let Some(landing) = &self.landing {
            landing.await_since(self.built_at_frame_start);
        }
    }

    /// Queues every pipeline the cache file records, as the first frame of
    /// an updated build does, and returns whether any is left to build: for
    /// a renderer that prepares an app's pipelines without drawing.
    #[cfg(not(target_arch = "wasm32"))]
    pub(crate) fn prepare_recorded(&mut self) -> bool {
        self.effect_renderer.warm_after_first_frame();
        if let Some(records) = self.recent_after_update.take() {
            self.warm_recent(&records);
        }
        self.compiling()
    }

    /// Whether the background compiler has a queued pipeline left to build.
    #[cfg(not(target_arch = "wasm32"))]
    pub(crate) fn compiling(&self) -> bool {
        self.landing
            .as_ref()
            .is_some_and(|landing| landing.in_flight())
    }

    /// Whether the last frame drew a placeholder and pipelines are still
    /// compiling, so its picture is not final yet.
    pub(crate) fn awaits_pipelines(&self) -> bool {
        #[cfg(not(target_arch = "wasm32"))]
        if let Some(landing) = &self.landing {
            return self.drew_placeholder && landing.in_flight();
        }
        self.drew_placeholder
    }

    /// The compiler's account of the pipelines it builds, for a runtime that
    /// reads it off the present thread.
    #[cfg(not(target_arch = "wasm32"))]
    pub(crate) fn landing(&self) -> Option<Arc<crate::pipeline_compiler::Landing>> {
        self.landing.clone()
    }

    /// Calls `wake` from the compiling thread when a pipeline a placeholder
    /// waited for lands, so an app with nothing else to draw draws again.
    #[cfg(not(target_arch = "wasm32"))]
    pub(crate) fn wake_on_landing(&self, wake: Box<dyn Fn() + Send + Sync>) {
        if let Some(landing) = &self.landing {
            landing.wake_with(wake);
        }
    }

    pub fn debug_cpu_allocation_stats(&self) -> DebugCpuAllocationStats {
        DebugCpuAllocationStats {
            scene_graph_node_count: 0,
            scene_graph_heap_bytes: 0,
            scene_hits_len: 0,
            scene_hits_cap: 0,
            scene_node_index_len: 0,
            scene_node_index_cap: 0,
            text_renderer_pool_len: self.text_image_cache.len(),
            text_renderer_pool_cap: self.text_image_cache.cap().get(),
            image_texture_cache_len: self.image_texture_cache.len(),
            image_texture_cache_cap: self.image_texture_cache.cap().get(),
            run_arena_staging_bytes: self.run_store.arena_staging_bytes(),
            run_store_bytes: self.run_store.stored_bytes(),
            run_store_runs: self.run_store.stored_count(),
            scratch_image_vertices_cap: self.scratch_image_vertices.capacity(),
            scratch_image_indices_cap: self.scratch_image_indices.capacity(),
            scratch_image_cmds_cap: self.scratch_image_cmds.capacity(),
            scratch_glyph_instances_cap: self.scratch_glyph_instances.plain.capacity()
                + self.scratch_glyph_instances.turned.capacity(),
            layer_cache_len: self.layer_cache.len(),
            layer_cache_bytes: self.layer_cache.bytes(),
        }
    }
    pub fn render_to_rgba_pixels(
        &mut self,
        width: u32,
        height: u32,
        packet: FramePacket,
        surface_epoch: u64,
        returns: &mut RenderReturns,
    ) -> Result<Vec<u8>, String> {
        if width == 0 || height == 0 {
            return Err("Screenshot size must be non-zero".to_string());
        }
        let errors_before = self.device_errors.error_count();

        let output_texture = crate::offscreen::create_2d_texture(
            &self.device,
            wgpu::TextureFormat::Rgba8Unorm,
            width,
            height,
            wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC,
            Some("Screenshot Output Texture"),
        );
        let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default());
        self.render_internal(
            width,
            height,
            packet,
            surface_epoch,
            returns,
            OutputMode::Screenshot,
            Some(&output_view),
            None,
        )?;
        if returns.outcome != PresentOutcome::Presented {
            return Err(format!(
                "Screenshot rendering did not complete: {:?}",
                returns.outcome
            ));
        }
        self.read_screenshot_pixels(&output_texture, errors_before)
    }

    fn read_screenshot_pixels(
        &mut self,
        output_texture: &wgpu::Texture,
        errors_before: u64,
    ) -> Result<Vec<u8>, String> {
        let width = output_texture.width();
        let height = output_texture.height();
        let bytes_per_pixel = 4u32;
        let unpadded_bytes_per_row = width
            .checked_mul(bytes_per_pixel)
            .ok_or_else(|| "Screenshot row byte size overflow".to_string())?;
        let padded_bytes_per_row =
            align_to(unpadded_bytes_per_row, wgpu::COPY_BYTES_PER_ROW_ALIGNMENT);
        let output_buffer_size = padded_bytes_per_row as u64 * height as u64;

        let output_buffer = self.device.create_buffer(&wgpu::BufferDescriptor {
            label: Some("Screenshot Readback Buffer"),
            size: output_buffer_size,
            usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
            mapped_at_creation: false,
        });

        let device = self.device.clone();
        let queue = self.queue.clone();
        let mut graph = WgpuFrameGraph::new(Some("Screenshot Copy Encoder"));
        let source = graph.import_surface("screenshot-copy-source");
        graph.add_fallible_command_pass(Some("Screenshot Copy Pass"), &[source], &[], |context| {
            context.encoder.copy_texture_to_buffer(
                wgpu::TexelCopyTextureInfo {
                    texture: output_texture,
                    mip_level: 0,
                    origin: wgpu::Origin3d::ZERO,
                    aspect: wgpu::TextureAspect::All,
                },
                wgpu::TexelCopyBufferInfo {
                    buffer: &output_buffer,
                    layout: wgpu::TexelCopyBufferLayout {
                        offset: 0,
                        bytes_per_row: Some(padded_bytes_per_row),
                        rows_per_image: Some(height),
                    },
                },
                wgpu::Extent3d {
                    width,
                    height,
                    depth_or_array_layers: 1,
                },
            );
            Ok(())
        });
        let mut executor = std::mem::take(&mut self.frame_graph_executor);
        let execution = executor.execute_recorded_graph(&device, &queue, graph);
        self.frame_graph_executor = executor;
        self.recall_frame_uploads();
        let execution = execution.map_err(|error| error.to_string())?;
        let submission_index = execution.submission;
        let copy_stats = execution.stats;
        self.last_frame_stats = self
            .last_frame_stats
            .map(|snapshot| snapshot.with_command_stats_added(copy_stats));

        let buffer_slice = output_buffer.slice(..);
        let (tx, rx) = mpsc::channel();
        buffer_slice.map_async(wgpu::MapMode::Read, move |result| {
            let _ = tx.send(result);
        });
        self.device
            .poll(wgpu::PollType::Wait {
                submission_index: Some(submission_index),
                timeout: None,
            })
            .map_err(|error| format!("Screenshot GPU wait failed: {error}"))?;

        match rx.recv_timeout(Duration::from_secs(3)) {
            Ok(Ok(())) => {}
            Ok(Err(err)) => return Err(format!("Screenshot map_async failed: {err:?}")),
            Err(err) => return Err(format!("Screenshot readback timed out: {err}")),
        }
        if self.device_errors.error_count() != errors_before {
            output_buffer.unmap();
            return Err("GPU errors occurred while capturing the screenshot".to_string());
        }

        let mapped = buffer_slice
            .get_mapped_range()
            .map_err(|err| format!("Screenshot readback could not be read: {err}"))?;
        let mut pixels = vec![0u8; (width as usize) * (height as usize) * 4];

        let src_row_len = padded_bytes_per_row as usize;
        let dst_row_len = unpadded_bytes_per_row as usize;
        for row in 0..height as usize {
            let src_offset = row * src_row_len;
            let dst_offset = row * dst_row_len;
            pixels[dst_offset..dst_offset + dst_row_len]
                .copy_from_slice(&mapped[src_offset..src_offset + dst_row_len]);
        }
        drop(mapped);
        output_buffer.unmap();

        self.convert_surface_pixels_to_rgba(&pixels)
    }

    /// Asks the buffers the submitted frame unmapped back for later frames.
    fn recall_frame_uploads(&mut self) {
        self.viewport_uniforms.uploads.recall();
        self.run_store.recall();
    }

    fn render_graph(
        &mut self,
        root_target: &Rc<OffscreenTarget>,
        packet: FramePacket,
        returns: &mut RenderReturns,
        output_mode: OutputMode,
        output: Option<(&wgpu::TextureView, &wgpu::BindGroup)>,
    ) -> Result<(), String> {
        let device = self.device.clone();
        let queue = self.queue.clone();
        let graph_start = Instant::now();
        let FramePacket {
            root,
            overlay,
            root_scale,
            clear,
            ..
        } = packet;
        let page = Rc::clone(root_target);

        #[cfg(not(target_arch = "wasm32"))]
        let (result, submitted) = {
            let mut executor = std::mem::take(&mut self.frame_graph_executor);
            let mut frame_graph = WgpuFrameGraph::new(Some("Renderer Frame Graph"));
            let surface = frame_graph.import_surface("renderer-surface");
            frame_graph.add_fallible_recorded_command_pass(
                Some("Renderer Frame Pass"),
                &[],
                &[surface],
                |frame_encoder| {
                    self.encode_frame(
                        frame_encoder,
                        &root,
                        overlay.as_ref(),
                        Rc::clone(&page),
                        root_scale,
                        clear,
                        output_mode,
                        output,
                    )
                },
            );
            let after_build = Instant::now();
            let execution = executor.execute_recorded_graph(&device, &queue, frame_graph);
            let after_execute = Instant::now();
            self.frame_graph_executor = executor;
            self.recall_frame_uploads();
            if let Some(total_ms) = should_log_wgpu_render_stage(graph_start, after_execute) {
                log::warn!(
                    "[wgpu-render-stage:graph] total_ms={total_ms:.2} build_ms={:.2} execute_ms={:.2}",
                    instant_ms(graph_start, after_build),
                    instant_ms(after_build, after_execute),
                );
            }
            match execution {
                Ok(execution) => {
                    if execution.stats.pass_count > 0 {
                        self.frame_stats.record_command_stats(execution.stats);
                    }
                    (Ok(()), true)
                }
                Err(crate::frame_graph::FrameGraphError::NoDeclaredPasses) => (Ok(()), false),
                Err(error) => (Err(error.to_string()), false),
            }
        };

        #[cfg(target_arch = "wasm32")]
        let (result, submitted) = {
            let mut executor = std::mem::take(&mut self.frame_graph_executor);
            let (result, execution) = {
                let mut frame_encoder =
                    executor.begin(&device, &queue, Some("Renderer Frame Encoder"));
                let initial_pass_count = frame_encoder.recorded_pass_count();
                let result = self.encode_frame(
                    &mut frame_encoder,
                    &root,
                    overlay.as_ref(),
                    Rc::clone(&page),
                    root_scale,
                    clear,
                    output_mode,
                    output,
                );
                let execution =
                    if result.is_ok() && frame_encoder.recorded_pass_count() > initial_pass_count {
                        Some(frame_encoder.finish())
                    } else {
                        None
                    };
                (result, execution)
            };
            let after_execute = Instant::now();
            self.frame_graph_executor = executor;
            if let Some(total_ms) = should_log_wgpu_render_stage(graph_start, after_execute) {
                log::warn!("[wgpu-render-stage:graph] total_ms={total_ms:.2}",);
            }
            let submitted = execution.is_some();
            if let Some(execution) = execution {
                self.frame_stats.record_command_stats(execution.stats);
            }
            (result, submitted)
        };
        #[cfg(not(target_arch = "wasm32"))]
        if submitted {
            self.recorder.note_frame_drawn();
            self.effect_renderer.warm_after_first_frame();
            if let Some(records) = self.recent_after_update.take() {
                self.warm_recent(&records);
            }
        }
        if !submitted {
            self.run_store.invalidate_uploads();
            self.text_glyph_gpu_run_cache.clear();
            self.text_glyph_run_arena = GlyphRunArena::new(self.copy_free);
        }
        returns.scene = Some(FrameSceneStorage { root, overlay });
        result
    }

    /// Records the frame: the root and overlay layer scenes into the frame's
    /// target, the output conversion when the target is not the presented
    /// image, and the viewport uniforms the recorded passes claimed.
    #[expect(clippy::too_many_arguments)]
    fn encode_frame<C: FrameCommandRecorder>(
        &mut self,
        recorder: &mut C,
        root: &LayerScene,
        overlay: Option<&LayerScene>,
        page: Rc<OffscreenTarget>,
        root_scale: f32,
        clear: wgpu::Color,
        output_mode: OutputMode,
        output: Option<(&wgpu::TextureView, &wgpu::BindGroup)>,
    ) -> Result<(), String> {
        FrameExecutor::new(self, recorder).render_frame(
            root,
            overlay,
            page,
            root_scale,
            wgpu::LoadOp::Clear(clear),
        )?;
        if let Some((output_view, bind_group)) = output {
            match output_mode {
                OutputMode::Display => &self.output_converter,
                OutputMode::Screenshot => &self.screenshot_converter,
            }
            .encode(
                &self.device,
                recorder,
                output_view,
                bind_group,
                self.adapter_backend,
                &self.recorder,
            );
            recorder.record_pass();
        }
        self.stage_frame_uploads(recorder);
        self.shape_pipelines.request_wanted();
        self.depth_gate.end_frame();
        self.pipeline_compiler.end_frame();
        self.viewport_uniforms.uploads.finish_frame();
        self.run_store.finish_frame();
        Ok(())
    }

    pub(crate) fn stage_frame_uploads(&mut self, recorder: &mut impl FrameCommandRecorder) {
        let mut upload = self.viewport_uniforms.stage_pending(&self.device, recorder);
        upload += self.run_store.stage_pending(&self.device, recorder);
        upload += self
            .text_glyph_run_arena
            .stage_pending(&self.device, recorder);
        self.run_store.fill_arena_trig(&self.device, recorder);
        self.frame_stats.record_command_stats(upload);
    }
    /// Claims this frame's next viewport uniform slot for `params`.
    pub(crate) fn claim_uniform_slot(&mut self, params: ViewportUniformParams) -> usize {
        let uniforms = Uniforms::of(params, PlacementData::zeroed(), RunTier::Arena);
        self.viewport_uniforms
            .claim(&self.device, &self.uniform_bind_group_layout, &uniforms)
    }

    /// Resolves a blurred shadow at `z` into a texture and queues its
    /// composites. The shadow's shapes and texts render into a source the
    /// size of their blur footprint, blur in place and take the post-blur
    /// cutouts; the source is then blitted over the rect the shadow covers.
    /// Shape-only shadows live in the shadow cache, keyed by their content
    /// and device placement, so a scrolling card re-blits its cached blur.
    /// The blurred shadow texture and whether the cache held it: a
    /// shape-only shadow is cached by content and placement, a shadow with
    /// text renders every frame.
    fn blurred_shadow_source<C: FrameCommandRecorder>(
        &mut self,
        recorder: &mut C,
        shadow: &ShadowDraw,
        source_device: DevicePixelBounds,
        pixel_radius: f32,
        root_scale: f32,
        transients: &mut Vec<(FrameTextureDescriptor, Rc<OffscreenTarget>)>,
    ) -> Option<(Rc<OffscreenTarget>, bool, SourceContent)> {
        let shape_only = shadow.texts.is_empty();
        let key = if shape_only {
            shape_shadow_surface_cache_key(shadow, source_device, pixel_radius, root_scale)
        } else {
            None
        };
        let content = key.map_or(SourceContent::Transient, |key| {
            SourceContent::retained(&key)
        });
        if let Some(entry) = key.and_then(|key| self.shadow_surface_cache.get(&key)) {
            return Some((Rc::clone(&entry.target), true, content));
        }
        // A shadow whose blur compiles is left out until the blur lands.
        if !self.effect_renderer.blur_draws_now(
            &self.device,
            TileMode::Decal,
            (source_device.width, source_device.height),
            pixel_radius,
            pixel_radius,
        ) {
            self.note_placeholder();
            return None;
        }
        if !shape_only {
            self.frame_stats.record_shadow_text_blur_fallback();
        }
        let source = self.render_shadow_source(
            recorder,
            shadow,
            source_device,
            pixel_radius,
            root_scale,
            key.is_some(),
            transients,
        )?;
        if let Some(key) = key {
            self.frame_stats
                .record_shadow_shape_cache_miss(source_device.width, source_device.height);
            self.frame_stats.maybe_print_shadow_shape_cache_miss(
                source_device.width,
                source_device.height,
                key.content_hash,
                pixel_radius,
                [source_device.x, source_device.y],
                shadow.shapes.as_ref().map_or(0, RunDraw::record_count) as usize,
                shadow.clip,
            );
            self.insert_cached_shadow_surface(key, Rc::clone(&source));
        }
        Some((source, false, content))
    }

    #[expect(clippy::too_many_arguments)]
    pub(crate) fn resolve_blurred_shadow<C: FrameCommandRecorder>(
        &mut self,
        recorder: &mut C,
        shadow: &ShadowDraw,
        z: usize,
        root_scale: f32,
        target_rect: DeviceRect4,
        transients: &mut Vec<(FrameTextureDescriptor, Rc<OffscreenTarget>)>,
        resolved: &mut Vec<ResolvedComposite>,
    ) {
        if !shadow.requires_surface()
            || skip_shadow_draws()
            || !root_scale.is_finite()
            || root_scale <= 0.0
        {
            return;
        }
        let Some(bounds) = shadow_draw_bounds(shadow) else {
            return;
        };
        let margin = blur_reach(shadow.blur_radius, root_scale);
        let source_bounds = expand_rect(bounds, margin, margin);
        let mut visible = source_bounds;
        if let Some(clip) = shadow.clip {
            let Some(clipped) = visible.intersect(expand_rect(clip, margin, margin)) else {
                return;
            };
            visible = clipped;
        }
        let target_logical = Rect {
            x: target_rect.0 / root_scale,
            y: target_rect.1 / root_scale,
            width: target_rect.2 / root_scale,
            height: target_rect.3 / root_scale,
        };
        let Some(visible) = visible.intersect(expand_rect(target_logical, margin, margin)) else {
            return;
        };
        let max_texture_dim = self.max_texture_dim();
        let shape_only = shadow.texts.is_empty();
        let anchor = shadow
            .shapes
            .as_ref()
            .and_then(|run| run.placement.snap_anchor);
        let source_device = shape_only
            .then(|| {
                translation_stable_anchored_device_pixel_bounds(
                    source_bounds,
                    anchor,
                    root_scale,
                    max_texture_dim,
                )
            })
            .flatten()
            .or_else(|| device_pixel_bounds(visible, root_scale, max_texture_dim));
        let Some(source_device) = source_device else {
            return;
        };
        let pixel_radius = shadow.blur_radius * root_scale;
        let Some((source, hit, content)) = self.blurred_shadow_source(
            recorder,
            shadow,
            source_device,
            pixel_radius,
            root_scale,
            transients,
        ) else {
            return;
        };
        let dest = (
            source_device.x,
            source_device.y,
            source_device.width as f32,
            source_device.height as f32,
        );
        let mut coverage = intersect_device_rects(dest, target_rect);
        if let Some(clip) = shadow.clip {
            coverage = coverage.and_then(|coverage| {
                intersect_device_rects(coverage, anchored_rect_to_device(clip, anchor, root_scale))
            });
        }
        let Some(coverage) = coverage else {
            return;
        };
        if hit {
            self.frame_stats.record_shadow_shape_cache_hit(
                (coverage.2 as u64).saturating_mul(coverage.3 as u64),
            );
        }
        let rounded_mask = shadow_composite_mask(shadow, anchor, root_scale);
        let downscaled =
            (source.width, source.height) != (source_device.width, source_device.height);
        let (sample_mode, source_viewport) = if downscaled {
            (
                CompositeSampleMode::Linear,
                Some((0.0, 0.0, source.width as f32, source.height as f32)),
            )
        } else {
            (CompositeSampleMode::Nearest, None)
        };
        resolved.push(ResolvedComposite {
            z_index: z,
            source,
            content,
            dest,
            scissor: Some(coverage),
            kind: ResolvedCompositeKind::Blit {
                alpha: 1.0,
                blend_mode: BlendMode::SrcOver,
                rounded_mask,
                sample_mode,
                source_viewport,
            },
        });
    }

    /// Draws a shadow's shapes and texts into a surface covering `bounds`
    /// and blurs it. A wide blur runs at its scratch size and its result
    /// stays there, read bilinearly by the composite; a post-blur cutout
    /// needs the surface's full size, so the blurred result is interpolated
    /// back into it first and the cutout drawn at that size. A retained
    /// result feeds the shadow cache; a transient one is registered with
    /// the frame's transients and released with them. `None` when the
    /// shadow draws nothing.
    #[expect(clippy::too_many_arguments)]
    fn render_shadow_source<C: FrameCommandRecorder>(
        &mut self,
        recorder: &mut C,
        shadow: &ShadowDraw,
        bounds: DevicePixelBounds,
        pixel_radius: f32,
        root_scale: f32,
        retained: bool,
        transients: &mut Vec<(FrameTextureDescriptor, Rc<OffscreenTarget>)>,
    ) -> Option<Rc<OffscreenTarget>> {
        let (width, height) = (bounds.width, bounds.height);
        let device = self.device.clone();
        let (scratch_width, scratch_height) =
            crate::effect_renderer::blur_scratch_size(pixel_radius, pixel_radius, width, height);
        let full_size_result = shadow.post_blur_cutouts.is_some()
            || (scratch_width, scratch_height) == (width, height);
        let (result_width, result_height) = if full_size_result {
            (width, height)
        } else {
            (scratch_width, scratch_height)
        };
        let result = if retained {
            Rc::new(self.acquire_retained_surface(result_width, result_height))
        } else {
            self.shadow_transient(
                recorder,
                transients,
                "Shadow Result",
                result_width,
                result_height,
            )
        };
        let source = if full_size_result {
            Rc::clone(&result)
        } else {
            self.shadow_transient(recorder, transients, "Shadow Source", width, height)
        };
        let offset = [bounds.x, bounds.y];
        let target = PassTarget {
            view: &source.view,
            width,
            height,
        };
        let scene = shadow_scene(
            shadow.shapes.as_ref(),
            &shadow.texts,
            blur_reach(shadow.blur_radius, root_scale),
        );
        let segment = PassSegment {
            scene: &scene,
            ops: &scene.draw_ops,
            composites: &[],
            offset,
            scissor: None,
            first_run_window: None,
            transform: SegmentTransform::IDENTITY,
            scale: root_scale,
        };
        let drew = self.encode_pass(
            recorder,
            target,
            std::slice::from_ref(&segment),
            wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
            "Shadow Source Pass",
        );
        match drew {
            Ok(true) => {}
            Ok(false) => {
                drop(source);
                if retained && let Ok(target) = Rc::try_unwrap(result) {
                    self.defer_offscreen_release(target);
                }
                return None;
            }
            Err(error) => {
                log::error!("shadow source pass failed: {error}");
                return None;
            }
        }
        if pixel_radius > 0.0 {
            let scratch_descriptor = self.transient_offscreen_descriptor(
                "Shadow Blur Scratch",
                scratch_width,
                scratch_height,
            );
            let scratch = recorder.acquire_transient_offscreen(&device, scratch_descriptor);
            let blurred = if full_size_result && (scratch_width, scratch_height) != (width, height)
            {
                Some(self.shadow_transient(
                    recorder,
                    transients,
                    "Shadow Blur Result",
                    scratch_width,
                    scratch_height,
                ))
            } else {
                None
            };
            let blur_dest = match &blurred {
                Some(blurred) => (&blurred.view, (scratch_width, scratch_height)),
                None => (&result.view, (result_width, result_height)),
            };
            let passes = self.effect_renderer.encode_blur_scissored_ping_pong_passes(
                recorder,
                &device,
                &source,
                &scratch,
                blur_dest,
                pixel_radius,
                pixel_radius,
                TileMode::Decal,
                None,
            );
            recorder.record_passes(passes);
            self.effect_renderer.record_blur_pass();
            recorder.release_transient_offscreen(scratch_descriptor, scratch);
            if let Some(blurred) = &blurred {
                self.effect_renderer
                    .encode_upscale_pass(recorder, &device, blurred, &result.view);
                recorder.record_pass();
            }
        }
        if let Some(cutout_run) = &shadow.post_blur_cutouts {
            let cutouts = shadow_scene(Some(cutout_run), &[], 0.0);
            let segment = PassSegment {
                scene: &cutouts,
                ops: &cutouts.draw_ops,
                composites: &[],
                offset,
                scissor: None,
                first_run_window: None,
                transform: SegmentTransform::IDENTITY,
                scale: root_scale,
            };
            if let Err(error) = self.encode_pass(
                recorder,
                target,
                std::slice::from_ref(&segment),
                wgpu::LoadOp::Load,
                "Shadow Cutout Pass",
            ) {
                log::error!("shadow cutout pass failed: {error}");
            }
        }
        Some(result)
    }

    /// A transient surface of a shadow's frame, released with the frame's
    /// transients.
    fn shadow_transient<C: FrameCommandRecorder>(
        &self,
        recorder: &mut C,
        transients: &mut Vec<(FrameTextureDescriptor, Rc<OffscreenTarget>)>,
        label: &'static str,
        width: u32,
        height: u32,
    ) -> Rc<OffscreenTarget> {
        let descriptor = self.transient_offscreen_descriptor(label, width, height);
        let target = Rc::new(recorder.acquire_transient_offscreen(&self.device, descriptor));
        transients.push((descriptor, Rc::clone(&target)));
        target
    }

    /// Whether `run` draws from retained buffers keyed by its command.
    pub(crate) fn run_is_stored(&self, run: &RunDraw) -> bool {
        self.run_store.is_stored(run)
    }

    /// The pipeline `segment` of a run at `placement` draws with. `clipped`
    /// says the variant tests the run's clip: a turned run's, since an
    /// unturned one's clip is its paint's scissor (see [`clip_scissor`]).
    fn run_pipeline_key(
        segment: &RecordSegment,
        placement: &crate::scene::Placement,
        tier: RunTier,
        ablation: ShapeAblation,
        turns: ShapeTurns,
        (depth, clip): (bool, SegmentClip),
        viewport: ViewportUniformParams,
    ) -> ShapePipelineKey {
        let blend_mode = supported_blend_mode(segment.blend);
        let laid = depth
            && blend_mode == BlendMode::SrcOver
            && !placement.paints()
            && clip != SegmentClip::Rounded
            && !ablation.material
            && !ablation.fill;
        ShapePipelineKey {
            blend_mode,
            tier,
            variant: ShapeVariant::of_segment(
                segment,
                clip,
                ablation,
                laid,
                (
                    turns == ShapeTurns::None,
                    turns == ShapeTurns::None
                        && placement.snap_anchor.is_some()
                        && viewport.offset.iter().all(|value| value.fract() == 0.0),
                ),
            ),
            turns,
            depth: if depth {
                ShapeDepth::Tested
            } else {
                ShapeDepth::Off
            },
        }
    }

    /// Queues for a later frame the shape pipelines `keys` name and the
    /// interiors they lay down, handing `awaited` each still to land.
    pub(crate) fn warm_shapes_later(
        &mut self,
        keys: &PipelineDemand,
        awaited: &mut impl FnMut(Pending<wgpu::RenderPipeline>),
    ) {
        for key in keys.keys() {
            self.shape_pipelines.warm_later(key, awaited);
        }
    }

    /// Prepares the pipeline of each key `pipelines` holds and, in a pass
    /// with a depth buffer, the one laying down its opaque interiors: the
    /// compiler's threads build the missing ones after the first while this
    /// thread builds that one.
    pub(crate) fn ensure_run_pipelines(&mut self, pipelines: &mut PipelineDemand) {
        self.shape_pipelines.prefetch(pipelines);
        for (key, vertices) in pipelines.drain() {
            self.ensure_shape_pipeline(key, vertices);
            if let Some(interior) = key.interior() {
                self.ensure_shape_pipeline(interior, vertices);
            }
        }
    }

    /// Brings a stored run's tables up to date and appends its draws, under
    /// a placement uniform of its own, to the pass's run draws `draws`, and
    /// the pipelines they draw with to `pipelines`.
    #[expect(clippy::too_many_arguments)]
    pub(crate) fn prepare_store_run<C: FrameCommandRecorder>(
        &mut self,
        recorder: &mut C,
        run: &RunDraw,
        viewport: ViewportUniformParams,
        root_scale: f32,
        window: &std::ops::Range<u32>,
        (depth, mut depth_keys): (bool, Option<&mut PipelineDemand>),
        (draws, pipelines): (&mut Vec<RunDrawCall>, &mut PipelineDemand),
    ) -> StoreRunBatch {
        let placement = &run.placement;
        let ablation = self.ablation.shape;
        let turns = ShapeTurns::of(viewport.transform, false);
        let scissor = clip_scissor(placement, root_scale, viewport);
        let clip = SegmentClip::of(placement, scissor.is_some());
        let start = draws.len();
        self.run_store.stored_run_draws(
            &self.device,
            run,
            &mut |segment| {
                let key = |depth| {
                    Self::run_pipeline_key(
                        segment,
                        placement,
                        RunTier::Store,
                        ablation,
                        turns,
                        (depth, clip),
                        viewport,
                    )
                };
                if let Some(keys) = depth_keys.as_deref_mut() {
                    keys.add(key(true), 0);
                }
                key(depth)
            },
            draws,
        );
        window_draws(draws, start, window);
        let run_draws = &draws[start..];
        let upload_start = Instant::now();
        let (upload, fill, tables) = self.run_store.upload_stored(
            &self.device,
            recorder,
            run,
            root_scale,
            window,
            run_draws,
        );
        if let Some(total_ms) = should_log_wgpu_render_stage(upload_start, Instant::now()) {
            log::warn!(
                "[wgpu-render-stage:run-upload] total_ms={total_ms:.2} bytes={} records={}",
                upload.upload_bytes,
                run.tables().shapes.len()
            );
        }
        self.frame_stats.record_command_stats(upload);
        if let Some(fill) = fill {
            self.frame_stats.add_shape_fill(fill);
        }
        let uniforms = Uniforms::of(
            viewport,
            PlacementData::of(&run.placement, root_scale, viewport.transform),
            RunTier::Store,
        );
        let uniform_slot =
            self.viewport_uniforms
                .claim(&self.device, &self.uniform_bind_group_layout, &uniforms);
        for draw in run_draws {
            pipelines.add(
                draw.key,
                crate::run_store::draw_vertices(
                    draw.records.end - draw.records.start,
                    draw.band_class,
                ),
            );
        }
        StoreRunBatch {
            tables,
            uniform_slot,
            draws: start..draws.len(),
            clip: scissor,
        }
    }

    pub(crate) fn open_arena(&mut self) -> usize {
        self.run_store.open_arena()
    }

    /// The records the open arena chunk holds so far.
    pub(crate) fn open_arena_records(&self) -> u32 {
        self.run_store.open_arena_records()
    }

    pub(crate) fn arena_accepts(&self, chunk: usize, run: &RunDraw) -> bool {
        self.run_store.arena_accepts(chunk, run)
    }

    /// Appends `window` of `run`'s records to the open arena chunk, their
    /// placement under `turn`, keyed for a pass whose flat and turned
    /// records alternate often when `mixed_turns`, and the pipelines they
    /// draw with to `pipelines`. `clip` says how its variant tests its clip:
    /// an unturned run leaves a rect clip to its paint's scissor, never a
    /// rounded one.
    #[expect(clippy::too_many_arguments)]
    pub(crate) fn append_arena_run(
        &mut self,
        chunk: usize,
        run: &RunDraw,
        window: std::ops::Range<u32>,
        root_scale: f32,
        viewport: ViewportUniformParams,
        mixed_turns: bool,
        (depth, clip): (bool, SegmentClip),
        (pipelines, mut depth_keys): (&mut PipelineDemand, Option<&mut PipelineDemand>),
    ) -> u32 {
        let placement = &run.placement;
        let ablation = self.ablation.shape;
        let turn = viewport.transform;
        let turns = ShapeTurns::of(turn, mixed_turns);
        self.run_store.append_arena(
            chunk,
            run,
            window,
            (root_scale, turn),
            &mut |segment| {
                let key = |depth| {
                    Self::run_pipeline_key(
                        segment,
                        placement,
                        RunTier::Arena,
                        ablation,
                        turns,
                        (depth, clip),
                        viewport,
                    )
                };
                if let Some(keys) = depth_keys.as_deref_mut() {
                    keys.add(key(true), 0);
                }
                key(depth)
            },
            pipelines,
        )
    }

    /// Moves the open chunk's draws since its last cut onto `out`, keeping
    /// it open.
    pub(crate) fn cut_arena(&mut self, chunk: usize, out: &mut Vec<RunDrawCall>) {
        self.run_store.cut_arena(&self.device, chunk, out);
    }

    /// Uploads the open chunk and moves its draws since its last cut onto
    /// `out`.
    pub(crate) fn close_arena(&mut self, chunk: usize, out: &mut Vec<RunDrawCall>) {
        if let Some(fill) = self.run_store.close_arena(&self.device, chunk, out) {
            self.frame_stats.add_shape_fill(fill);
        }
    }

    pub(crate) fn draw_run_calls(
        &self,
        pass: &mut TrackedPass<'_, '_>,
        tables: ArenaBinding<'_>,
        uniform_slot: usize,
        draws: &[RunDrawCall],
        scissor: (u32, u32, u32, u32),
        stage: RunStage,
    ) -> Result<(), String> {
        match stage {
            RunStage::Paint => {
                if draws.is_empty() {
                    return Ok(());
                }
                self.frame_stats.bump_shapes();
                self.record_run_draws(
                    pass,
                    tables,
                    uniform_slot,
                    scissor,
                    draws.iter().map(PlannedRunDraw::paint),
                )
            }
            RunStage::Interiors => {
                let planned = interior_run_draws(draws);
                if planned.is_empty() {
                    return Ok(());
                }
                self.frame_stats
                    .shape_interior_draws
                    .set(self.frame_stats.shape_interior_draws.get() + planned.len() as u32);
                self.record_run_draws(pass, tables, uniform_slot, scissor, planned.into_iter())
            }
        }
    }

    /// Records `planned` against one batch's tables, uniform and scissor.
    fn record_run_draws(
        &self,
        pass: &mut TrackedPass<'_, '_>,
        tables: ArenaBinding<'_>,
        uniform_slot: usize,
        scissor: (u32, u32, u32, u32),
        planned: impl ExactSizeIterator<Item = PlannedRunDraw>,
    ) -> Result<(), String> {
        self.frame_stats.add_draw_calls(planned.len() as u32);
        let (x, y, width, height) = scissor;
        pass.set_scissor_rect(x, y, width, height);
        self.viewport_uniforms.bind(pass, uniform_slot)?;
        pass.set_bind_group(1, tables.bind_group, &tables.offsets[2..]);
        for (slot, buffer) in tables.records.into_iter().enumerate() {
            pass.set_vertex_buffer(slot as u32, buffer.slice(u64::from(tables.offsets[slot])..));
        }
        let mut bound_class = None;
        for draw in planned {
            let key = draw.key;
            let (pipeline, fallback) = self
                .shape_pipelines
                .get(key)
                .ok_or_else(|| format!("shape pipeline {key:?} was not prepared"))?;
            if fallback {
                self.frame_stats
                    .shape_pipeline_fallback_draws
                    .set(self.frame_stats.shape_pipeline_fallback_draws.get() + 1);
            } else if !key.is_general() {
                self.frame_stats
                    .shape_specialized_draws
                    .set(self.frame_stats.shape_specialized_draws.get() + 1);
            }
            if bound_class != Some(draw.band_class) {
                pass.set_index_buffer(self.run_store.strip_index_buffer(draw.band_class).slice(..));
                bound_class = Some(draw.band_class);
            }
            pass.set_pipeline(pipeline);
            pass.draw_indexed(draw.indices, 0, draw.records);
        }
        Ok(())
    }

    /// Draws a stored-run batch, whose draws are its range of `draws`.
    pub(crate) fn draw_store_run(
        &self,
        pass: &mut TrackedPass<'_, '_>,
        batch: &StoreRunBatch,
        draws: &[RunDrawCall],
        scissor: (u32, u32, u32, u32),
        stage: RunStage,
    ) -> Result<(), String> {
        self.draw_run_calls(
            pass,
            batch.tables.binding(),
            batch.uniform_slot,
            &draws[batch.draws.clone()],
            scissor,
            stage,
        )
    }

    pub(crate) fn draw_arena(
        &self,
        pass: &mut TrackedPass<'_, '_>,
        chunk: usize,
        uniform_slot: usize,
        draws: &[RunDrawCall],
        scissor: (u32, u32, u32, u32),
        stage: RunStage,
    ) -> Result<(), String> {
        self.draw_run_calls(
            pass,
            self.run_store.arena_binding(chunk),
            uniform_slot,
            draws,
            scissor,
            stage,
        )
    }
    #[cfg(not(target_arch = "wasm32"))]
    pub(crate) fn surface_format(&self) -> wgpu::TextureFormat {
        self.display_format
    }

    pub fn device_error_count(&self) -> u64 {
        self.device_errors.error_count()
    }

    /// A pass that only applies `load_op` to the target, for a scene with
    /// nothing to draw that still needs its clear.
    pub(crate) fn clear_target<C: FrameCommandRecorder>(
        &self,
        recorder: &mut C,
        view: &wgpu::TextureView,
        load_op: wgpu::LoadOp<wgpu::Color>,
    ) {
        self.empty_pass(recorder, "Clear Pass", view, load_op);
    }

    pub(crate) fn empty_pass<C: FrameCommandRecorder>(
        &self,
        recorder: &mut C,
        label: &'static str,
        view: &wgpu::TextureView,
        load_op: wgpu::LoadOp<wgpu::Color>,
    ) {
        let pass = recorder.begin_color_pass(label, view, load_op);
        drop(pass);
        recorder.record_pass();
    }
    pub(crate) fn draw_image_cmds(
        &self,
        pass: &mut TrackedPass<'_, '_>,
        image_slot: &ImageSlot,
        uniform_slot: usize,
        cmds: &[ImageDrawCmd],
        (blend_mode, depth): (BlendMode, bool),
        bound: Option<(u32, u32, u32, u32)>,
    ) -> Result<(), String> {
        if cmds.is_empty() {
            return Ok(());
        }
        self.frame_stats.bump_images();
        self.frame_stats.add_draw_calls(cmds.len() as u32);
        let mut bound_pipeline = None;
        self.viewport_uniforms.bind(pass, uniform_slot)?;
        pass.set_index_buffer(image_slot.indices.slice());
        pass.set_vertex_buffer(0, image_slot.vertices.slice());
        let mut clips_bound = false;
        for cmd in cmds {
            let Some((x, y, width, height)) = bounded_scissor(cmd.scissor, bound) else {
                continue;
            };
            pass.set_scissor_rect(x, y, width, height);
            let variant = ImageVariant::of(cmd, depth);
            if bound_pipeline != Some(variant) {
                pass.set_pipeline(self.image_pipeline(blend_mode, variant));
                bound_pipeline = Some(variant);
            }
            pass.set_bind_group(1, &cmd.image.bind_group, &[]);
            let instance = match cmd.clip {
                Some(clip) => {
                    if !clips_bound {
                        let clips = image_slot
                            .clips
                            .as_ref()
                            .ok_or_else(|| "rounded image without clip instances".to_string())?;
                        pass.set_vertex_buffer(1, clips.slice());
                        clips_bound = true;
                    }
                    clip
                }
                None => 0,
            };
            pass.draw_indexed(
                cmd.index_start..(cmd.index_start + 6),
                0,
                instance..instance + 1,
            );
        }
        Ok(())
    }

    pub(crate) fn draw_glyph_cmds(
        &self,
        pass: &mut TrackedPass<'_, '_>,
        (glyph_slot, turned_slot): (Option<&BufferUpload>, Option<&BufferUpload>),
        uniform_slot: usize,
        cmds: &[GlyphDrawCmd],
        bound: Option<(u32, u32, u32, u32)>,
        frame: PassFrame,
    ) -> Result<(), String> {
        if cmds.is_empty() {
            return Ok(());
        }
        let whole_target = frame.scissor(bound);
        self.frame_stats.bump_text();
        let mut bound_atlas = None;
        // Which shared instances are bound, turned or plain, and which
        // pipeline; retained runs draw with a plain one.
        let mut shared_bound: Option<bool> = None;
        let mut bound_pipeline: Option<GlyphQuads> = None;
        let mut bound_run_instances: Option<&wgpu::Buffer> = None;
        let mut draws = 0u32;
        for draw in GlyphDraws::new(cmds) {
            let scissor = match draw.scissor {
                Some(scissor) => bounded_scissor(scissor, bound),
                None => Some(whole_target),
            };
            let Some((x, y, width, height)) = scissor else {
                continue;
            };
            pass.set_scissor_rect(x, y, width, height);
            if !bound_atlas.is_some_and(|atlas| Rc::ptr_eq(atlas, draw.atlas)) {
                pass.set_bind_group(1, draw.atlas.as_ref(), &[]);
                bound_atlas = Some(draw.atlas);
            }
            draws += 1;
            match draw.step {
                GlyphDrawStep::Shared(instances, quads) => {
                    if bound_pipeline != Some(quads) {
                        pass.set_pipeline(self.glyph_atlas_pipeline(frame.depth, quads));
                        bound_pipeline = Some(quads);
                    }
                    let turned = quads.turned();
                    if shared_bound != Some(turned) {
                        let slot =
                            if turned { turned_slot } else { glyph_slot }.ok_or_else(|| {
                                "shared glyph draw without glyph instances".to_string()
                            })?;
                        self.viewport_uniforms.bind(pass, uniform_slot)?;
                        pass.set_vertex_buffer(0, slot.slice());
                        shared_bound = Some(turned);
                        bound_run_instances = None;
                    }
                    pass.draw(0..GLYPH_QUAD_CORNERS, instances);
                }
                GlyphDrawStep::Retained {
                    run,
                    uniform_slot: retained_slot,
                    aligned,
                } => {
                    let quads = GlyphQuads::plain(aligned);
                    if bound_pipeline != Some(quads) {
                        pass.set_pipeline(self.glyph_atlas_pipeline(frame.depth, quads));
                        bound_pipeline = Some(quads);
                    }
                    shared_bound = None;
                    self.viewport_uniforms.bind(pass, retained_slot)?;
                    let instances = run.span.instance_buffer();
                    if bound_run_instances != Some(instances) {
                        pass.set_vertex_buffer(0, instances.slice(..));
                        bound_run_instances = Some(instances);
                    }
                    pass.draw(0..GLYPH_QUAD_CORNERS, run.span.instances());
                }
            }
        }
        self.frame_stats.add_draw_calls(draws);
        Ok(())
    }
    pub(crate) fn append_image_draw_cmd(
        &mut self,
        image_draw: &ImageDraw,
        viewport: ViewportUniformParams,
        root_scale: f32,
        (image_vertices, image_indices): (&mut Vec<Vertex>, &mut Vec<u32>),
        image_clips: &mut Vec<DeviceRoundedClip>,
        image_cmds: &mut Vec<ImageDrawCmd>,
    ) -> Result<(), String> {
        let snap_delta = image_draw
            .snap_anchor
            .map(|anchor| snap_delta_for_anchor(anchor, root_scale))
            .unwrap_or_default();
        let rect = image_draw.rect.translate(snap_delta.x, snap_delta.y);
        if rect.width <= 0.0 || rect.height <= 0.0 || image_draw.alpha <= 0.0 {
            return Ok(());
        }

        let (mut tint, cpu_filter) = tint_for_image(image_draw.color_filter, image_draw.alpha);
        if tint[3] <= 0.0 {
            return Ok(());
        }

        let filtered_image = cpu_filter
            .map(|filter| apply_filter_to_bitmap(&image_draw.image, filter))
            .transpose()?;
        let prepared_image = filtered_image.as_ref().unwrap_or(&image_draw.image);
        if let cranpose_ui_graphics::ImagePixelFormat::Alpha8 { color } = prepared_image.format() {
            for (channel, value) in tint[..3].iter_mut().zip(color) {
                *channel *= value as f32 / 255.0;
            }
        }

        let geometry = resolve_image_geometry_with_delta(image_draw, root_scale, snap_delta);
        let Some(scissor) =
            scissor_rect_for_layer(geometry.rect, geometry.clip, root_scale, viewport)
        else {
            return Ok(());
        };

        let Some(uv_rect) = image_uv_rect(&image_draw.image, image_draw.src_rect) else {
            return Ok(());
        };
        let device_quad = geometry.device_quad(root_scale);
        let index_start =
            push_image_quad((image_vertices, image_indices), device_quad, tint, &uv_rect);
        let clip = DeviceRoundedClip::of(image_draw, root_scale).map(|clip| {
            image_clips.push(clip);
            image_clips.len() as u32 - 1
        });
        image_cmds.push(ImageDrawCmd {
            index_start,
            scissor,
            image: self.image_binding(prepared_image, geometry.sampling)?,
            clip,
        });
        Ok(())
    }

    /// Uploads a pass's image and glyph quads into the frame's buffers.
    pub(crate) fn upload_image_slot<C: FrameCommandRecorder>(
        &self,
        recorder: &mut C,
        vertices: &[Vertex],
        indices: &[u32],
        clips: &[DeviceRoundedClip],
    ) -> ImageSlot {
        ImageSlot {
            vertices: recorder.upload_buffer(
                image_vertex_spec(),
                &self.device,
                bytemuck::cast_slice(vertices),
            ),
            indices: recorder.upload_buffer(
                image_index_spec(),
                &self.device,
                bytemuck::cast_slice(indices),
            ),
            clips: (!clips.is_empty()).then(|| {
                recorder.upload_buffer(
                    UploadAllocatorSpec::vertex(
                        "Image Clip Buffer",
                        std::mem::size_of::<DeviceRoundedClip>() as u64,
                    ),
                    &self.device,
                    bytemuck::cast_slice(clips),
                )
            }),
        }
    }

    pub(crate) fn upload_glyph_instances<C: FrameCommandRecorder>(
        &self,
        recorder: &mut C,
        instances: &[GlyphInstance],
    ) -> BufferUpload {
        recorder.upload_buffer(
            glyph_instance_spec(),
            &self.device,
            bytemuck::cast_slice(instances),
        )
    }

    pub(crate) fn upload_turned_glyphs<C: FrameCommandRecorder>(
        &self,
        recorder: &mut C,
        glyphs: &[TurnedGlyph],
    ) -> BufferUpload {
        recorder.upload_buffer(
            turned_glyph_spec(),
            &self.device,
            bytemuck::cast_slice(glyphs),
        )
    }

    fn glyph_atlas_entry_for(
        &mut self,
        glyph: &SoftwareGlyphAtlasGlyph,
    ) -> Result<GlyphAtlasEntry, String> {
        if let Some(entry) = self
            .text_glyph_atlas
            .upload_glyph(glyph, &mut self.frame_stats)
        {
            return Ok(entry);
        }

        // The draws queued so far bind the full atlas: the glyphs they placed
        // reach it before a new atlas takes its place.
        self.flush_glyph_uploads();
        self.text_glyph_atlas.reset(
            &self.device,
            &self.image_bind_group_layout,
            GlyphSamplers {
                nearest: &self.image_nearest_sampler,
                linear: &self.image_linear_sampler,
            },
        );
        Err("text glyph atlas filled and was reset".to_string())
    }

    /// Writes the glyphs the pass placed in the atlas to its texture, ahead
    /// of the submit that draws them.
    pub(crate) fn flush_glyph_uploads(&mut self) {
        self.text_glyph_atlas.flush_uploads(
            &self.queue,
            &mut self.frame_graph_executor,
            &mut self.frame_stats,
        );
    }

    fn glyph_atlas_entry_for_cached(
        &mut self,
        glyph: &SoftwareGlyphAtlasPlacement,
    ) -> Option<GlyphAtlasEntry> {
        let entry = self
            .text_glyph_atlas
            .entry(&GlyphAtlasSlotKey::new(glyph.key, glyph.color))?;
        self.frame_stats.record_text_glyph_atlas_hits(1);
        Some(entry)
    }

    fn glyph_atlas_entry_for_placement(
        &mut self,
        glyph: &SoftwareGlyphAtlasPlacement,
    ) -> Result<GlyphAtlasEntry, String> {
        if let Some(entry) = self.glyph_atlas_entry_for_cached(glyph) {
            return Ok(entry);
        }

        let Some(upload_glyph) = self.text_glyph_mask_cache.atlas_glyph_for_placement(glyph) else {
            return Err("text glyph placement has no retained raster mask".to_string());
        };
        self.glyph_atlas_entry_for(&upload_glyph)
    }

    /// The atlas entries of a run's drawing glyphs: of `cached_glyph_run`,
    /// which holds only those, or of the drawing glyphs of `collected_run`,
    /// a run just collected, whose new glyphs are uploaded here.
    fn prepare_text_glyph_entries(
        &mut self,
        run_key: TextGlyphRunCacheKey,
        atlas_generation: u64,
        cached_glyph_run: Option<&RunGlyphs>,
        collected_run: &[SoftwareGlyphAtlasRunGlyph],
        entries: &mut Vec<GlyphAtlasEntry>,
    ) -> Result<Rc<[GlyphAtlasEntry]>, String> {
        entries.clear();
        let mut seen = RunAtlasEntries::new();
        if let Some(glyph_run) = cached_glyph_run {
            for glyph in glyph_run.iter() {
                entries.push(seen.entry(self, &glyph, |renderer| {
                    renderer.glyph_atlas_entry_for_placement(&glyph)
                })?);
            }
        } else {
            for run_glyph in collected_run {
                entries.push(seen.entry(
                    self,
                    &run_glyph.placement(),
                    |renderer| match run_glyph {
                        SoftwareGlyphAtlasRunGlyph::Cached(placement) => {
                            renderer.glyph_atlas_entry_for_placement(placement)
                        }
                        SoftwareGlyphAtlasRunGlyph::New(glyph) => {
                            renderer.glyph_atlas_entry_for(glyph)
                        }
                    },
                )?);
            }
        }

        let entries: Rc<[GlyphAtlasEntry]> = Rc::from(entries.as_slice());
        if let Some(cached) = self.text_glyph_run_cache.get_mut(&run_key) {
            cached.atlas_entries = Some(Rc::clone(&entries));
            cached.atlas_generation = atlas_generation;
        }
        Ok(entries)
    }

    #[expect(clippy::too_many_arguments)]
    fn append_text_glyph_quad_run(
        &mut self,
        source_raster_rect: Rect,
        quads: GlyphRunQuads<'_>,
        (clip, cut): (Option<Rect>, Option<[f32; 4]>),
        viewport: ViewportUniformParams,
        root_scale: f32,
        glyph_instances: &mut GlyphInstances,
        record_cached_hits: bool,
    ) -> (GlyphQuads, GlyphExtent) {
        let start = glyph_instances.lens();
        let mut extent = GlyphExtent::EMPTY;
        let origin = [source_raster_rect.x, source_raster_rect.y];
        let bounds = quads.bounds.at(origin);
        let kind = if viewport.transform.is_identity()
            && cut.is_none_or(|edges| {
                bounds[0] >= edges[0]
                    && bounds[1] >= edges[1]
                    && bounds[2] <= edges[2]
                    && bounds[3] <= edges[3]
            })
            && quads
                .bounds
                .within_viewport(source_raster_rect, clip, viewport, root_scale)
        {
            // Every quad shows whole and uncut: they sit at whole pixels from
            // the run's origin, which then decides their alignment, and
            // their extent is the run's bounds.
            glyph_instances.plain.extend(quads.instances_at(origin));
            extent.add(bounds);
            GlyphQuads::plain(glyph_rect_aligned(
                [origin[0], origin[1], origin[0], origin[1]],
                viewport.offset,
            ))
        } else if viewport.transform.is_identity() {
            let plain = &mut glyph_instances.plain;
            let mut aligned = true;
            for_each_visible_glyph(
                source_raster_rect,
                quads,
                (clip, cut),
                viewport,
                root_scale,
                |glyph| {
                    aligned &= glyph_rect_aligned(glyph.rect, viewport.offset);
                    extent.add(glyph.rect);
                    plain.push(glyph);
                },
            );
            GlyphQuads::plain(aligned)
        } else {
            let turn = viewport.transform;
            let turned = &mut glyph_instances.turned;
            for_each_visible_glyph(
                source_raster_rect,
                quads,
                (clip, cut),
                viewport,
                root_scale,
                |glyph| turned.push(TurnedGlyph::new(glyph, turn)),
            );
            GlyphQuads::Turned
        };
        let end = glyph_instances.lens();
        let appended = (end.0 - start.0) + (end.1 - start.1);
        if record_cached_hits {
            self.frame_stats
                .record_text_glyph_atlas_hits(u32::try_from(appended).unwrap_or(u32::MAX));
        }
        (kind, extent)
    }

    /// The viewport a retained glyph run draws under: its vertices sit at
    /// its raster rect's origin, so an untransformed target moves its offset
    /// back by that origin, and a transformed one adds the origin to each
    /// vertex before its transform, as the shared path's vertices hold it.
    fn retained_glyph_viewport(
        viewport: ViewportUniformParams,
        source_raster_rect: Rect,
    ) -> ViewportUniformParams {
        if !viewport.transform.is_identity() {
            return ViewportUniformParams {
                origin: [source_raster_rect.x, source_raster_rect.y],
                ..viewport
            };
        }
        ViewportUniformParams {
            offset: [
                viewport.offset[0] - source_raster_rect.x,
                viewport.offset[1] - source_raster_rect.y,
            ],
            ..viewport
        }
    }

    fn retained_text_glyph_run(
        &mut self,
        cache_key: TextGlyphRunCacheKey,
    ) -> Option<Rc<CachedGpuTextGlyphRun>> {
        let atlas_generation = self.text_glyph_atlas.generation();
        let frame = self.text_glyph_run_frame;
        self.text_glyph_gpu_run_cache
            .get(&cache_key)
            .filter(|cached| cached.atlas_generation == atlas_generation)
            .inspect(|cached| cached.last_frame.set(frame))
            .cloned()
    }

    /// Opens a frame for text runs: runs no frame drew for
    /// [`TEXT_GLYPH_GPU_RUN_IDLE_FRAMES`] leave the GPU cache and those no
    /// frame drew for [`TEXT_GLYPH_RUN_IDLE_FRAMES`] the CPU cache, and the
    /// arena takes back the quads dropped retained runs held.
    fn begin_text_glyph_run_frame(&mut self) {
        #[cfg(feature = "installed-fonts")]
        if let Some(fonts) = self.text_fonts.with_installed_faces() {
            // Text drawn before the install may show a missing glyph where an
            // installed face now has one.
            self.text_fonts = fonts;
            self.text_image_cache.clear();
            self.text_glyph_run_cache.clear();
            self.text_glyph_gpu_run_cache.clear();
        }
        self.text_glyph_run_frame += 1;
        let frame = self.text_glyph_run_frame;
        let demand = std::mem::take(&mut self.text_glyph_runs_drawn).saturating_mul(2);
        self.text_glyph_run_cache.set_cap(
            std::num::NonZeroUsize::new(demand.max(MIN_TEXT_GLYPH_RUN_CACHE_ITEMS))
                .unwrap_or(std::num::NonZeroUsize::MIN),
        );
        evict_idle(
            &mut self.text_glyph_gpu_run_cache,
            frame,
            TEXT_GLYPH_GPU_RUN_IDLE_FRAMES,
            |run| run.last_frame.get(),
        );
        evict_idle(
            &mut self.text_glyph_run_cache,
            frame,
            TEXT_GLYPH_RUN_IDLE_FRAMES,
            |run| run.last_frame.get(),
        );
        self.text_glyph_run_arena.begin_frame();
        let evacuating = self.text_glyph_run_arena.evacuating();
        if !evacuating.is_empty() {
            let moving: Vec<TextGlyphRunCacheKey> = self
                .text_glyph_gpu_run_cache
                .iter()
                .filter(|(_, run)| evacuating.contains(&run.span.chunk()))
                .map(|(key, _)| *key)
                .collect();
            for key in &moving {
                self.text_glyph_gpu_run_cache.pop(key);
            }
        }
    }

    fn emit_retained_text_glyph_run_if_ready(
        &mut self,
        cache_key: TextGlyphRunCacheKey,
        quads: GlyphRunQuads<'_>,
        viewport: ViewportUniformParams,
        source_raster_rect: Rect,
        scissor: (u32, u32, u32, u32),
        glyph_cmds: &mut Vec<GlyphDrawCmd>,
    ) -> bool {
        let Some(run) = self.retained_text_glyph_run(cache_key).or_else(|| {
            if self.ensure_retained_text_glyph_run(cache_key, quads) {
                self.retained_text_glyph_run(cache_key)
            } else {
                None
            }
        }) else {
            return false;
        };
        let uniform_slot =
            self.claim_uniform_slot(Self::retained_glyph_viewport(viewport, source_raster_rect));
        self.frame_stats
            .record_text_glyph_atlas_hits(u32::try_from(quads.len()).unwrap_or(u32::MAX));
        // The run's quads sit at whole texels from its raster origin.
        let aligned = viewport.transform.is_identity()
            && glyph_rect_aligned(
                [source_raster_rect.x, source_raster_rect.y, 0.0, 0.0],
                [viewport.offset[0], viewport.offset[1]],
            );
        glyph_cmds.push(GlyphDrawCmd::retained(
            (run, uniform_slot, aligned),
            scissor,
            self.text_glyph_atlas.bind_group(viewport.transform),
        ));
        true
    }

    fn ensure_retained_text_glyph_run(
        &mut self,
        cache_key: TextGlyphRunCacheKey,
        quads: GlyphRunQuads<'_>,
    ) -> bool {
        let atlas_generation = self.text_glyph_atlas.generation();
        if self
            .text_glyph_gpu_run_cache
            .peek(&cache_key)
            .is_some_and(|cached| cached.atlas_generation == atlas_generation)
        {
            return true;
        }

        let Some(span) = self
            .text_glyph_run_arena
            .insert(&self.device, quads.instances_at([0.0, 0.0]))
        else {
            return false;
        };
        self.text_glyph_gpu_run_cache.put(
            cache_key,
            Rc::new(CachedGpuTextGlyphRun {
                span,
                atlas_generation,
                last_frame: Cell::new(self.text_glyph_run_frame),
            }),
        );
        true
    }
    /// Appends the glyph atlas draws of `text_draw`, whose logical geometry
    /// `logical` the caller found drawing into a rect `viewport` shows.
    /// `false` when the text cannot draw from the atlas (animated motion, or
    /// a run the atlas cannot hold): nothing was appended, and the caller
    /// draws the text as a rasterized image instead.
    pub(crate) fn append_text_glyph_draws(
        &mut self,
        (text_draw, logical): (&TextDraw, (Rect, f32)),
        viewport: ViewportUniformParams,
        root_scale: f32,
        glyph_instances: &mut GlyphInstances,
        glyph_cmds: &mut Vec<GlyphDrawCmd>,
    ) -> bool {
        let (_, raster_rect, _, text_scale, static_text_motion) =
            text_raster_geometry_at(text_draw, logical, root_scale);
        if !static_text_motion {
            return false;
        }
        let run_key = Self::text_glyph_run_cache_key(text_draw, raster_rect, text_scale);
        let mut collected_run = std::mem::take(&mut self.scratch_text_glyph_run);
        let mut generated_entries = std::mem::take(&mut self.scratch_text_glyph_entries);
        let initial_instance_len = glyph_instances.lens();
        let initial_cmd_len = glyph_cmds.len();
        let drew = self
            .text_glyph_run(
                text_draw,
                raster_rect,
                text_scale,
                run_key,
                &mut collected_run,
            )
            .is_some_and(|run| {
                self.append_text_glyph_run(
                    TextGlyphRunDraw {
                        text_draw,
                        raster_rect,
                        run_key,
                        run,
                    },
                    (viewport, root_scale),
                    (&collected_run, &mut generated_entries),
                    glyph_instances,
                    glyph_cmds,
                )
            });
        self.scratch_text_glyph_run = collected_run;
        self.scratch_text_glyph_entries = generated_entries;
        if !drew {
            glyph_instances.truncate(initial_instance_len);
            glyph_cmds.truncate(initial_cmd_len);
        }
        drew
    }

    /// The run of `text_draw` from the cache, or collected into
    /// `collected_run` and cached; `None` when the text cannot draw from the
    /// atlas.
    fn text_glyph_run(
        &mut self,
        text_draw: &TextDraw,
        raster_rect: Rect,
        text_scale: f32,
        run_key: TextGlyphRunCacheKey,
        collected_run: &mut Vec<SoftwareGlyphAtlasRunGlyph>,
    ) -> Option<TextGlyphRunLookup> {
        let atlas_generation = self.text_glyph_atlas.generation();
        let frame = self.text_glyph_run_frame;
        self.text_glyph_runs_drawn += 1;
        if let Some(cached) = self.text_glyph_run_cache.get(&run_key) {
            cached.last_frame.set(frame);
            return Some(TextGlyphRunLookup {
                glyphs: cached.glyphs.clone(),
                bounds: cached.bounds,
                cached: true,
                entries: (cached.atlas_generation == atlas_generation)
                    .then(|| cached.atlas_entries.as_ref().map(Rc::clone))
                    .flatten(),
            });
        }
        collected_run.clear();
        if collect_solid_text_atlas_run(
            text_draw.text.string().as_ref(),
            raster_rect,
            &text_draw.text_style,
            text_draw.color,
            text_draw.font_size,
            text_scale,
            &self.text_fonts,
            &mut self.text_glyph_mask_cache,
            collected_run,
        )
        .is_none()
        {
            log_text_atlas_fallback(text_draw);
            return None;
        }
        let Some(glyphs) = RunGlyphs::of(
            collected_run
                .iter()
                .map(SoftwareGlyphAtlasRunGlyph::placement),
            &mut self.run_glyph_scratch,
        ) else {
            log_text_atlas_fallback(text_draw);
            return None;
        };
        let bounds = GlyphRunBounds::of(glyphs.iter());
        self.frame_stats.record_text_glyph_run_collected();
        // A run this frame drew stays: the cache grows rather than evict it.
        let cache = &mut self.text_glyph_run_cache;
        if cache.len() == cache.cap().get()
            && cache
                .peek_lru()
                .is_some_and(|(_, run)| run.last_frame.get() == frame)
        {
            cache.set_cap(cache.cap().saturating_add(cache.len()));
        }
        self.text_glyph_run_cache.put(
            run_key,
            CachedTextGlyphRun {
                glyphs: glyphs.clone(),
                bounds,
                atlas_entries: None,
                atlas_generation: 0,
                last_frame: Cell::new(frame),
            },
        );
        Some(TextGlyphRunLookup {
            glyphs,
            bounds,
            cached: false,
            entries: None,
        })
    }

    /// Appends the draws of a text's run: its retained quads when it has
    /// them, else its visible quads into the frame's shared glyphs. `false`
    /// when the atlas cannot hold its glyphs.
    fn append_text_glyph_run(
        &mut self,
        draw: TextGlyphRunDraw<'_>,
        (viewport, root_scale): (ViewportUniformParams, f32),
        (collected_run, generated_entries): (
            &[SoftwareGlyphAtlasRunGlyph],
            &mut Vec<GlyphAtlasEntry>,
        ),
        glyph_instances: &mut GlyphInstances,
        glyph_cmds: &mut Vec<GlyphDrawCmd>,
    ) -> bool {
        let TextGlyphRunDraw {
            text_draw,
            raster_rect,
            run_key,
            run,
        } = draw;
        let draw_rect = Rect {
            x: raster_rect.x / root_scale,
            y: raster_rect.y / root_scale,
            width: raster_rect.width / root_scale,
            height: raster_rect.height / root_scale,
        };
        let Some(scissor) = scissor_rect_for_layer(draw_rect, text_draw.clip, root_scale, viewport)
        else {
            return true;
        };
        let texel = self.text_glyph_atlas.texel();

        // A retained run's quads are uploaded whole: under a turn only the
        // shared path cuts them to a clip.
        let clip_needs_cut = !viewport.transform.is_identity() && text_draw.clip.is_some();
        if let Some(entries) = run.entries.as_ref()
            && entries.len() >= RETAINED_TEXT_GLYPH_RUN_MIN_QUADS
            && !clip_needs_cut
            && self.emit_retained_text_glyph_run_if_ready(
                run_key,
                GlyphRunQuads {
                    glyphs: &run.glyphs,
                    entries,
                    texel,
                    bounds: run.bounds,
                },
                viewport,
                raster_rect,
                scissor,
                glyph_cmds,
            )
        {
            return true;
        }

        let turned = !viewport.transform.is_identity();
        let instance_start = glyph_instances.len_of(turned);
        let (entries, entries_cached) = match run.entries {
            Some(entries) => (entries, true),
            None => {
                let Ok(entries) = self.prepare_text_glyph_entries(
                    run_key,
                    self.text_glyph_atlas.generation(),
                    run.cached.then_some(&run.glyphs),
                    collected_run,
                    generated_entries,
                ) else {
                    return false;
                };
                (entries, false)
            }
        };
        let quads = GlyphRunQuads {
            glyphs: &run.glyphs,
            entries: &entries,
            texel: self.text_glyph_atlas.texel(),
            bounds: run.bounds,
        };
        let cut = glyph_cut_edges(text_draw.clip, scissor, viewport, root_scale);
        let (kind, extent) = self.append_text_glyph_quad_run(
            raster_rect,
            quads,
            (text_draw.clip, cut),
            viewport,
            root_scale,
            glyph_instances,
            entries_cached,
        );
        let instance_end = glyph_instances.len_of(turned);
        if instance_end > instance_start {
            let (clip, bounds) = if turned {
                (None, scissor)
            } else {
                shared_glyph_clip(extent, scissor, viewport)
            };
            glyph_cmds.push(GlyphDrawCmd::shared(
                (instance_start..instance_end, kind),
                clip,
                bounds,
                self.text_glyph_atlas.bind_group(viewport.transform),
            ));
        }
        true
    }

    #[expect(clippy::too_many_arguments)]
    fn append_image_bitmap_draw_cmd(
        &mut self,
        image: &ImageBitmap,
        rect: Rect,
        clip: Option<Rect>,
        sampling: ImageSampling,
        viewport: ViewportUniformParams,
        root_scale: f32,
        image_vertices: &mut Vec<Vertex>,
        image_indices: &mut Vec<u32>,
        image_cmds: &mut Vec<ImageDrawCmd>,
    ) -> Result<(), String> {
        if rect.width <= 0.0 || rect.height <= 0.0 {
            return Ok(());
        }

        let (device_quad, scissor_rect) =
            if sampling == ImageSampling::Nearest && root_scale.is_finite() && root_scale > 0.0 {
                let left_px = (rect.x * root_scale).round();
                let top_px = (rect.y * root_scale).round();
                let width_px = (rect.width * root_scale).round().max(1.0);
                let height_px = (rect.height * root_scale).round().max(1.0);
                let snapped_rect = Rect {
                    x: left_px / root_scale,
                    y: top_px / root_scale,
                    width: width_px / root_scale,
                    height: height_px / root_scale,
                };
                let right_px = left_px + width_px;
                let bottom_px = top_px + height_px;
                (
                    [
                        [left_px, top_px],
                        [right_px, top_px],
                        [left_px, bottom_px],
                        [right_px, bottom_px],
                    ],
                    snapped_rect,
                )
            } else {
                (
                    rect_to_quad(rect).map(|[x, y]| [x * root_scale, y * root_scale]),
                    rect,
                )
            };

        let Some(scissor) = scissor_rect_for_layer(scissor_rect, clip, root_scale, viewport) else {
            return Ok(());
        };
        let Some(uv_rect) = image_uv_rect(image, None) else {
            return Ok(());
        };

        let index_start = push_image_quad(
            (image_vertices, image_indices),
            device_quad,
            [1.0, 1.0, 1.0, 1.0],
            &uv_rect,
        );
        image_cmds.push(ImageDrawCmd {
            index_start,
            scissor,
            image: self.image_binding(image, sampling)?,
            clip: None,
        });
        Ok(())
    }

    /// Appends `text_draw` as a rasterized image if `viewport` shows it.
    pub(crate) fn append_text_image_draw_cmds(
        &mut self,
        text_draw: &TextDraw,
        viewport: ViewportUniformParams,
        root_scale: f32,
        image_vertices: &mut Vec<Vertex>,
        image_indices: &mut Vec<u32>,
        image_cmds: &mut Vec<ImageDrawCmd>,
    ) -> Result<(), String> {
        let Some((logical_rect, raster_rect, clip, text_scale, static_text_motion)) =
            self.text_raster_geometry(text_draw, root_scale)
        else {
            return Ok(());
        };
        if !text_draw_is_visible_in_viewport(logical_rect, clip, viewport, root_scale) {
            return Ok(());
        }

        let raster_source = self.text_image_raster_source(
            text_draw,
            logical_rect,
            raster_rect,
            clip,
            root_scale,
            static_text_motion,
        );
        let source_draw = raster_source.draw.as_ref();
        let source_raster_rect = raster_source.raster_rect;
        let source_origin = if source_draw.text.string().span_styles().is_empty() {
            Point::new(source_raster_rect.x, source_raster_rect.y)
        } else {
            Point::new(raster_rect.x, raster_rect.y)
        };

        let mut cache_key = Self::text_image_cache_key(
            source_draw,
            source_raster_rect,
            text_scale,
            static_text_motion,
        );
        if !source_draw.text.string().span_styles().is_empty() {
            let mut state = default_hash::new();
            cache_key.0.hash(&mut state);
            (source_origin.x - source_raster_rect.x)
                .to_bits()
                .hash(&mut state);
            (source_origin.y - source_raster_rect.y)
                .to_bits()
                .hash(&mut state);
            cache_key = TextImageCacheKey(state.finish());
        }
        let image = if let Some(cached) = self.text_image_cache.get(&cache_key) {
            self.frame_stats
                .record_text_image_cache_hit(cached.image.width(), cached.image.height());
            cached.image.clone()
        } else {
            let Some(image) = self.rasterize_text_draw_to_image(
                source_draw,
                source_raster_rect,
                source_origin,
                text_scale,
            ) else {
                return Ok(());
            };
            self.frame_stats
                .record_text_image_cache_miss(image.width(), image.height());
            self.text_image_cache.put(
                cache_key,
                CachedTextImage {
                    image: image.clone(),
                },
            );
            image
        };

        let draw_origin = if static_text_motion {
            Point::new(
                source_raster_rect.x / root_scale,
                source_raster_rect.y / root_scale,
            )
        } else {
            Point::new(logical_rect.x, logical_rect.y)
        };
        let draw_rect = Rect {
            x: draw_origin.x,
            y: draw_origin.y,
            width: image.width() as f32 / root_scale,
            height: image.height() as f32 / root_scale,
        };
        self.append_image_bitmap_draw_cmd(
            &image,
            draw_rect,
            clip,
            sampling_under(ImageSampling::Nearest, viewport.transform),
            viewport,
            root_scale,
            image_vertices,
            image_indices,
            image_cmds,
        )
    }

    fn text_image_raster_source<'a>(
        &mut self,
        text_draw: &'a TextDraw,
        logical_rect: Rect,
        raster_rect: Rect,
        clip: Option<Rect>,
        root_scale: f32,
        static_text_motion: bool,
    ) -> TextRasterSource<'a> {
        let Some(clip) = clip else {
            return TextRasterSource {
                draw: Cow::Borrowed(text_draw),
                raster_rect,
            };
        };
        if !static_text_motion || text_draw.text.string().text().find('\n').is_none() {
            return TextRasterSource {
                draw: Cow::Borrowed(text_draw),
                raster_rect,
            };
        }

        if !text_draw.text.string().span_styles().is_empty() {
            let device_clip = Rect {
                x: (clip.x * root_scale).floor(),
                y: (clip.y * root_scale).floor(),
                width: ((clip.x + clip.width) * root_scale).ceil() - (clip.x * root_scale).floor(),
                height: ((clip.y + clip.height) * root_scale).ceil()
                    - (clip.y * root_scale).floor(),
            };
            return TextRasterSource {
                draw: Cow::Borrowed(text_draw),
                raster_rect: raster_rect.intersect(device_clip).unwrap_or(raster_rect),
            };
        }

        let line_starts = self
            .text_line_index_cache
            .line_starts(text_draw.text.string());
        clipped_text_raster_source_with_line_starts(
            text_draw,
            logical_rect,
            raster_rect,
            clip,
            root_scale,
            line_starts.as_ref(),
        )
    }

    fn text_raster_geometry(
        &self,
        text_draw: &TextDraw,
        root_scale: f32,
    ) -> Option<(Rect, Rect, Option<Rect>, f32, bool)> {
        text_raster_geometry_for_draw(text_draw, root_scale)
    }

    fn text_image_cache_key(
        text_draw: &TextDraw,
        raster_rect: Rect,
        text_scale: f32,
        static_text_motion: bool,
    ) -> TextImageCacheKey {
        let mut state = default_hash::new();
        hash_text_draw_for_cache(text_draw, raster_rect, text_scale, &mut state);
        hash_text_raster_geometry_for_cache(raster_rect, static_text_motion, &mut state);
        TextImageCacheKey(state.finish())
    }

    /// The key of a static text's glyph run. A run places its glyphs from
    /// the text's origin and aligns its lines within the text block, so the
    /// size of the rect it draws in does not change it: only an empty rect,
    /// which draws nothing, does.
    fn text_glyph_run_cache_key(
        text_draw: &TextDraw,
        raster_rect: Rect,
        text_scale: f32,
    ) -> TextGlyphRunCacheKey {
        let mut state = default_hash::new();
        hash_text_draw_for_cache(text_draw, raster_rect, text_scale, &mut state);
        (raster_rect.width <= 0.0 || raster_rect.height <= 0.0).hash(&mut state);
        TextGlyphRunCacheKey(state.finish())
    }

    fn rasterize_text_draw_to_image(
        &mut self,
        text_draw: &TextDraw,
        raster_rect: Rect,
        source_origin: Point,
        text_scale: f32,
    ) -> Option<ImageBitmap> {
        rasterize_annotated_text_region(
            text_draw.text.string().as_ref(),
            raster_rect,
            if text_draw.text.string().span_styles().is_empty() {
                Point::new(raster_rect.x, raster_rect.y)
            } else {
                source_origin
            },
            &text_draw.text_style,
            text_draw.color,
            text_draw.font_size,
            text_scale,
            &self.text_fonts,
            Some(&mut self.text_glyph_mask_cache),
        )
    }
}

struct TextRasterSource<'a> {
    draw: Cow<'a, TextDraw>,
    raster_rect: Rect,
}

fn clipped_text_raster_source_with_line_starts<'a>(
    text_draw: &'a TextDraw,
    logical_rect: Rect,
    raster_rect: Rect,
    clip: Rect,
    root_scale: f32,
    line_starts: &[usize],
) -> TextRasterSource<'a> {
    if line_starts.len() < MIN_MULTILINE_TEXT_LINES_FOR_CLIPPED_RASTER {
        return TextRasterSource {
            draw: Cow::Borrowed(text_draw),
            raster_rect,
        };
    }

    let Some(visible_rect) = logical_rect.intersect(clip) else {
        return TextRasterSource {
            draw: Cow::Borrowed(text_draw),
            raster_rect,
        };
    };

    let line_count = line_starts.len().max(1);
    let line_height = logical_rect.height / line_count as f32;
    if !line_height.is_finite() || line_height <= 0.0 {
        return TextRasterSource {
            draw: Cow::Borrowed(text_draw),
            raster_rect,
        };
    }

    let visible_top = ((visible_rect.y - logical_rect.y) / line_height).floor() as isize;
    let visible_bottom =
        ((visible_rect.y + visible_rect.height - logical_rect.y) / line_height).ceil() as isize;
    let start_line = visible_top.saturating_sub(1).max(0) as usize;
    let end_line = (visible_bottom + 1).max(start_line as isize + 1) as usize;
    let end_line = end_line.min(line_count);
    if start_line == 0 && end_line >= line_count {
        return TextRasterSource {
            draw: Cow::Borrowed(text_draw),
            raster_rect,
        };
    }

    let byte_start = line_starts[start_line];
    let byte_end = line_end_offset(text_draw.text.string().text(), line_starts, end_line - 1);
    if byte_start >= byte_end {
        return TextRasterSource {
            draw: Cow::Borrowed(text_draw),
            raster_rect,
        };
    }

    let slice_y = logical_rect.y + start_line as f32 * line_height;
    let slice_height = (end_line - start_line) as f32 * line_height;
    let mut slice_raster_rect = Rect {
        x: logical_rect.x * root_scale,
        y: slice_y * root_scale,
        width: logical_rect.width * root_scale,
        height: slice_height * root_scale,
    };
    slice_raster_rect.x = slice_raster_rect.x.round();
    slice_raster_rect.y = slice_raster_rect.y.round();
    slice_raster_rect.width = slice_raster_rect.width.ceil().max(1.0);
    slice_raster_rect.height = slice_raster_rect.height.ceil().max(1.0);

    let mut sliced_draw = text_draw.clone();
    sliced_draw.rect = Rect {
        x: logical_rect.x,
        y: slice_y,
        width: logical_rect.width,
        height: slice_height,
    };
    sliced_draw.text = DrawnText::of(
        Arc::new(text_draw.text.string().subsequence(byte_start..byte_end)),
        &text_draw.text_style,
    );

    TextRasterSource {
        draw: Cow::Owned(sliced_draw),
        raster_rect: slice_raster_rect,
    }
}

fn line_start_offsets(text: &str) -> Vec<usize> {
    let mut starts =
        Vec::with_capacity(text.as_bytes().iter().filter(|b| **b == b'\n').count() + 1);
    starts.push(0);
    starts.extend(
        text.char_indices()
            .filter_map(|(index, ch)| (ch == '\n').then_some(index + ch.len_utf8())),
    );
    starts
}

fn line_end_offset(text: &str, line_starts: &[usize], line: usize) -> usize {
    line_starts.get(line + 1).copied().unwrap_or(text.len())
}

fn align_to(value: u32, alignment: u32) -> u32 {
    debug_assert!(alignment > 0);
    value.div_ceil(alignment) * alignment
}

impl GpuRenderer {
    fn convert_surface_pixels_to_rgba(&self, pixels: &[u8]) -> Result<Vec<u8>, String> {
        if !pixels.len().is_multiple_of(4) {
            return Err("Screenshot readback has an incomplete pixel".to_string());
        }
        Ok(pixels.to_vec())
    }
}

/// The scissor of a logical rect in a target whose origin sits at
/// `viewport.offset` of the scene's device space, clamped to the target.
/// `None` when nothing of the rect lands in the target.
pub(crate) fn scissor_rect_for_rect(
    rect: Rect,
    root_scale: f32,
    viewport: ViewportUniformParams,
) -> Option<(u32, u32, u32, u32)> {
    let width = viewport.width as f32;
    let height = viewport.height as f32;
    let mut left = canonicalize_device_coordinate(rect.x * root_scale);
    let mut top = canonicalize_device_coordinate(rect.y * root_scale);
    let mut right = canonicalize_device_coordinate((rect.x + rect.width) * root_scale);
    let mut bottom = canonicalize_device_coordinate((rect.y + rect.height) * root_scale);
    if !viewport.transform.is_identity() {
        let bounds = viewport.transform.target_bounds(Rect {
            x: left,
            y: top,
            width: right - left,
            height: bottom - top,
        });
        (left, top) = (bounds.x, bounds.y);
        (right, bottom) = (bounds.x + bounds.width, bounds.y + bounds.height);
    }
    let left = (left - viewport.offset[0]).clamp(0.0, width).floor();
    let top = (top - viewport.offset[1]).clamp(0.0, height).floor();
    let right = (right - viewport.offset[0]).clamp(0.0, width).ceil();
    let bottom = (bottom - viewport.offset[1]).clamp(0.0, height).ceil();
    if right <= left || bottom <= top {
        return None;
    }
    Some((
        left as u32,
        top as u32,
        (right - left) as u32,
        (bottom - top) as u32,
    ))
}

fn scissor_rect_for_layer(
    rect: Rect,
    clip: Option<Rect>,
    root_scale: f32,
    viewport: ViewportUniformParams,
) -> Option<(u32, u32, u32, u32)> {
    let clipped_rect = match clip {
        Some(clip_rect) => rect.intersect(clip_rect)?,
        None => rect,
    };
    scissor_rect_for_rect(clipped_rect, root_scale, viewport)
}
fn tint_for_image(
    color_filter: Option<ColorFilter>,
    alpha: f32,
) -> ([f32; 4], Option<ColorFilter>) {
    let alpha = alpha.clamp(0.0, 1.0);
    match color_filter {
        Some(filter) if filter.supports_gpu_vertex_modulation() => {
            let Some(tint) = filter.gpu_vertex_tint() else {
                return ([1.0, 1.0, 1.0, alpha], Some(filter));
            };
            (
                [
                    tint[0].clamp(0.0, 1.0),
                    tint[1].clamp(0.0, 1.0),
                    tint[2].clamp(0.0, 1.0),
                    (tint[3] * alpha).clamp(0.0, 1.0),
                ],
                None,
            )
        }
        Some(filter) => ([1.0, 1.0, 1.0, alpha], Some(filter)),
        None => ([1.0, 1.0, 1.0, alpha], None),
    }
}

/// The rounded clip an image takes in place, in device pixels: one instance
/// its draw reads, beside the quad's vertices.
#[repr(C)]
#[derive(Clone, Copy, Debug, Pod, Zeroable)]
pub(crate) struct DeviceRoundedClip {
    rect: [f32; 4],
    radius: f32,
}

impl DeviceRoundedClip {
    const ATTRIBS: [wgpu::VertexAttribute; 2] = wgpu::vertex_attr_array![
        4 => Float32x4,
        5 => Float32,
    ];

    fn desc() -> wgpu::VertexBufferLayout<'static> {
        wgpu::VertexBufferLayout {
            array_stride: std::mem::size_of::<Self>() as wgpu::BufferAddress,
            step_mode: wgpu::VertexStepMode::Instance,
            attributes: &Self::ATTRIBS,
        }
    }

    /// The rounded clip `image` takes at `root_scale`, scaled as a run's
    /// placement scales its clip.
    fn of(image: &ImageDraw, root_scale: f32) -> Option<Self> {
        let rounding = image.rounding.filter(|_| image.clip.is_some())?;
        let device =
            crate::geometry::device_rect(rounding.rect, root_scale, image.snap_anchor.is_some());
        Some(Self {
            rect: [device.x, device.y, device.width, device.height],
            radius: rounding.radius * root_scale,
        })
    }
}

/// Appends an image quad over `device_quad` and its two triangles, and
/// returns the index its triangles start at.
fn push_image_quad(
    (vertices, indices): (&mut Vec<Vertex>, &mut Vec<u32>),
    device_quad: [[f32; 2]; 4],
    color: [f32; 4],
    uv_rect: &ImageUvRect,
) -> u32 {
    let base_vertex = vertices.len() as u32;
    let index_start = indices.len() as u32;
    indices.extend_from_slice(&[
        base_vertex,
        base_vertex + 1,
        base_vertex + 2,
        base_vertex + 2,
        base_vertex + 1,
        base_vertex + 3,
    ]);
    let corners = [
        [uv_rect.min[0], uv_rect.min[1]],
        [uv_rect.max[0], uv_rect.min[1]],
        [uv_rect.min[0], uv_rect.max[1]],
        [uv_rect.max[0], uv_rect.max[1]],
    ];
    vertices.extend(
        device_quad
            .into_iter()
            .zip(corners)
            .map(|(position, uv)| Vertex {
                position,
                color,
                uv,
                uv_bounds: uv_rect.sample_bounds,
            }),
    );
    index_start
}

fn image_uv_rect(image: &ImageBitmap, src_rect: Option<Rect>) -> Option<ImageUvRect> {
    let Some(src) = src_rect else {
        return Some(ImageUvRect {
            min: [0.0, 0.0],
            max: [1.0, 1.0],
            sample_bounds: [0.0, 0.0, 1.0, 1.0],
        });
    };

    let (u_min, u_max, u_bound_min, u_bound_max) =
        source_axis_uv(src.x, src.width, image.width() as f32)?;
    let (v_min, v_max, v_bound_min, v_bound_max) =
        source_axis_uv(src.y, src.height, image.height() as f32)?;

    Some(ImageUvRect {
        min: [u_min, v_min],
        max: [u_max, v_max],
        sample_bounds: [u_bound_min, v_bound_min, u_bound_max, v_bound_max],
    })
}

/// `entry`'s rect in an atlas whose texel is `texel` of its side, an
/// exact power of two: the products are the quotients by the side.
fn glyph_atlas_uv_rect(entry: GlyphAtlasEntry, texel: f32) -> ImageUvRect {
    let min = [entry.x as f32 * texel, entry.y as f32 * texel];
    let max = [
        (entry.x + entry.width) as f32 * texel,
        (entry.y + entry.height) as f32 * texel,
    ];
    let center_min = [
        (entry.x as f32 + 0.5) * texel,
        (entry.y as f32 + 0.5) * texel,
    ];
    let center_max = [
        (entry.x as f32 + entry.width as f32 - 0.5).max(entry.x as f32 + 0.5) * texel,
        (entry.y as f32 + entry.height as f32 - 0.5).max(entry.y as f32 + 0.5) * texel,
    ];
    ImageUvRect {
        min,
        max,
        sample_bounds: [center_min[0], center_min[1], center_max[0], center_max[1]],
    }
}

#[derive(Clone, Copy)]
pub(crate) struct ResolvedImageGeometry {
    pub(crate) rect: Rect,
    pub(crate) clip: Option<Rect>,
    pub(crate) sampling: ImageSampling,
    quad: [[f32; 2]; 4],
    snap_anchored: bool,
}

pub(crate) fn resolve_image_geometry(image: &ImageDraw, root_scale: f32) -> ResolvedImageGeometry {
    let snap_delta = image
        .snap_anchor
        .map(|anchor| snap_delta_for_anchor(anchor, root_scale))
        .unwrap_or_default();
    resolve_image_geometry_with_delta(image, root_scale, snap_delta)
}

fn resolve_image_geometry_with_delta(
    image: &ImageDraw,
    root_scale: f32,
    snap_delta: Point,
) -> ResolvedImageGeometry {
    let mut rect = image.rect.translate(snap_delta.x, snap_delta.y);
    let mut quad = translate_quad(image.quad, snap_delta);
    snap_nearest_image_to_device_pixels(image.sampling, &mut rect, &mut quad, root_scale);
    ResolvedImageGeometry {
        rect,
        clip: image.clip,
        sampling: image.sampling,
        quad,
        snap_anchored: image.snap_anchor.is_some(),
    }
}

impl ResolvedImageGeometry {
    pub(crate) fn device_quad(self, root_scale: f32) -> [[f32; 2]; 4] {
        nearest_image_device_quad(self.sampling, self.quad, root_scale).unwrap_or_else(|| {
            if self.snap_anchored {
                canonicalized_scaled_quad(self.quad, root_scale)
            } else {
                scaled_quad(self.quad, root_scale)
            }
        })
    }
}

fn snap_nearest_image_to_device_pixels(
    sampling: ImageSampling,
    rect: &mut Rect,
    quad: &mut [[f32; 2]; 4],
    root_scale: f32,
) {
    if sampling != ImageSampling::Nearest || !root_scale.is_finite() || root_scale <= 0.0 {
        return;
    }

    let Some(quad_rect) = axis_aligned_quad_rect(*quad) else {
        return;
    };

    let left_px = (quad_rect.x * root_scale).round();
    let top_px = (quad_rect.y * root_scale).round();
    let width_px = (quad_rect.width * root_scale).round().max(1.0);
    let height_px = (quad_rect.height * root_scale).round().max(1.0);
    let snapped = Rect {
        x: left_px / root_scale,
        y: top_px / root_scale,
        width: width_px / root_scale,
        height: height_px / root_scale,
    };

    *rect = snapped;
    *quad = crate::rect_to_quad(snapped);
}

fn nearest_image_device_quad(
    sampling: ImageSampling,
    quad: [[f32; 2]; 4],
    root_scale: f32,
) -> Option<[[f32; 2]; 4]> {
    if sampling != ImageSampling::Nearest || !root_scale.is_finite() || root_scale <= 0.0 {
        return None;
    }

    let rect = axis_aligned_quad_rect(quad)?;
    let left_px = (rect.x * root_scale).round();
    let top_px = (rect.y * root_scale).round();
    let width_px = (rect.width * root_scale).round().max(1.0);
    let height_px = (rect.height * root_scale).round().max(1.0);
    let right_px = left_px + width_px;
    let bottom_px = top_px + height_px;
    Some([
        [left_px, top_px],
        [right_px, top_px],
        [left_px, bottom_px],
        [right_px, bottom_px],
    ])
}

fn source_axis_uv(start: f32, extent: f32, image_extent: f32) -> Option<(f32, f32, f32, f32)> {
    if !start.is_finite()
        || !extent.is_finite()
        || !image_extent.is_finite()
        || extent == 0.0
        || image_extent <= 0.0
    {
        return None;
    }

    let end = start + extent;
    let edge_min = start.min(end).clamp(0.0, image_extent);
    let edge_max = start.max(end).clamp(0.0, image_extent);
    if edge_max <= edge_min {
        return None;
    }

    let center_min = edge_min + 0.5;
    let center_max = edge_max - 0.5;
    let (bound_min, bound_max) = if center_min <= center_max {
        (center_min, center_max)
    } else {
        let center = (edge_min + edge_max) * 0.5;
        (center, center)
    };

    Some((
        edge_min / image_extent,
        edge_max / image_extent,
        bound_min / image_extent,
        bound_max / image_extent,
    ))
}

fn apply_filter_to_bitmap(image: &ImageBitmap, filter: ColorFilter) -> Result<ImageBitmap, String> {
    let pixel_count = image.width() as usize * image.height() as usize;
    let mut filtered = Vec::with_capacity(pixel_count * 4);
    for index in 0..pixel_count {
        let pixel = image.rgba8_pixel(index);
        let rgba = [
            pixel[0] as f32 / 255.0,
            pixel[1] as f32 / 255.0,
            pixel[2] as f32 / 255.0,
            pixel[3] as f32 / 255.0,
        ];
        let out = filter.apply_rgba(rgba);
        filtered.push((out[0].clamp(0.0, 1.0) * 255.0).round() as u8);
        filtered.push((out[1].clamp(0.0, 1.0) * 255.0).round() as u8);
        filtered.push((out[2].clamp(0.0, 1.0) * 255.0).round() as u8);
        filtered.push((out[3].clamp(0.0, 1.0) * 255.0).round() as u8);
    }
    ImageBitmap::from_rgba8(image.width(), image.height(), filtered)
        .map_err(|error| format!("failed to build filtered bitmap: {error}"))
}

/// How a text raster samples its texels under a segment transform: as it
/// asks on the pixel grid, and filtered once a transform turns it off the
/// grid, where no texel lands on a pixel for nearest sampling to keep.
fn sampling_under(sampling: ImageSampling, transform: SegmentTransform) -> ImageSampling {
    if transform.is_identity() {
        sampling
    } else {
        ImageSampling::Linear
    }
}

/// The rounded mask a shadow's composite applies, in the scene's device
/// pixels: an inner shadow masks itself to its fill shape, and a shadow
/// lowered out of a clipped layer masks itself to that layer's rounded clip.
pub(crate) fn shadow_composite_mask(
    shadow: &ShadowDraw,
    snap_anchor: Option<SnapAnchor>,
    root_scale: f32,
) -> Option<RoundedCompositeMask> {
    inner_shadow_composite_mask(shadow, root_scale).or_else(|| {
        shadow.rounded_clip.map(|clip| RoundedCompositeMask {
            rect: mask_rect(anchored_device_rect(clip.rect, snap_anchor, root_scale)),
            radii: clip.radii.map(|radius| radius * root_scale),
        })
    })
}

/// A scene holding just a shadow's own draws, in the order they arrive, so
/// the shadow source renders through the same pass encoder as everything
/// else.
fn shadow_scene(
    shapes: Option<&RunDraw>,
    texts: &[TextDraw],
    input_margin: f32,
) -> CompositorScene {
    let mut scene = CompositorScene::new();
    if let Some(run) = shapes {
        scene.push_run(run.clone());
    }
    for text in texts {
        let z_index = scene.next_z();
        scene.draw_ops.push(DrawOp {
            z_index,
            kind: DrawOpKind::Text(scene.texts.len()),
        });
        let mut source_text = text.clone();
        source_text.clip = source_text
            .clip
            .map(|clip| expand_rect(clip, input_margin, input_margin));
        scene.texts.push(source_text);
        scene.next_z += 1;
    }
    scene
}

/// The whole device pixels a logical rect covers, or `None` when it covers
/// none or more than a texture can hold.
fn device_pixel_bounds(
    rect: Rect,
    root_scale: f32,
    max_texture_dim: u32,
) -> Option<DevicePixelBounds> {
    let x = (rect.x * root_scale).floor();
    let y = (rect.y * root_scale).floor();
    let right = ((rect.x + rect.width) * root_scale).ceil();
    let bottom = ((rect.y + rect.height) * root_scale).ceil();
    let width = (right - x).max(0.0) as u32;
    let height = (bottom - y).max(0.0) as u32;
    if width == 0 || height == 0 || width > max_texture_dim || height > max_texture_dim {
        return None;
    }
    Some(DevicePixelBounds {
        x,
        y,
        width,
        height,
    })
}
fn inner_shadow_composite_mask(
    shadow: &ShadowDraw,
    root_scale: f32,
) -> Option<RoundedCompositeMask> {
    let run = shadow.shapes.as_ref()?;
    if !run
        .tables()
        .shapes
        .iter()
        .any(|record| record.blend_mode() == BlendMode::DstOut)
    {
        return None;
    }
    let fill = run.tables().shapes.get(0)?;
    let rect = run.placement.translated_bounds(fill.stored_rect());
    if rect.width <= 0.0 || rect.height <= 0.0 {
        return None;
    }
    let resolved =
        cranpose_ui_graphics::RoundedCornerShape::with_radii(cranpose_ui_graphics::CornerRadii {
            top_left: fill.radii[0],
            top_right: fill.radii[1],
            bottom_right: fill.radii[2],
            bottom_left: fill.radii[3],
        })
        .resolve(rect.width, rect.height);
    let radii = [
        resolved.top_left * root_scale,
        resolved.top_right * root_scale,
        resolved.bottom_left * root_scale,
        resolved.bottom_right * root_scale,
    ];

    Some(RoundedCompositeMask {
        rect: mask_rect(anchored_device_rect(
            rect,
            run.placement.snap_anchor,
            root_scale,
        )),
        radii,
    })
}

/// One draw call of a run batch's stage: its pipeline, the strip index
/// buffer it binds and the indices and records it instances.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct PlannedRunDraw {
    pub(crate) key: ShapePipelineKey,
    pub(crate) band_class: u8,
    pub(crate) indices: std::ops::Range<u32>,
    pub(crate) records: std::ops::Range<u32>,
}

impl PlannedRunDraw {
    /// A draw of the paint, as it was recorded.
    fn paint(draw: &RunDrawCall) -> Self {
        Self {
            key: draw.key,
            band_class: draw.band_class,
            indices: draw.indices(),
            records: draw.records.clone(),
        }
    }
}

/// The draw calls laying a batch's opaque interiors down: last first, so
/// they go down front to back, one call over each stretch of neighbouring
/// draws that share the interior pipeline, cut to the span from the stretch's
/// first draw holding an occluder to its last. The draws between ride along:
/// the pipeline lays any record's exact interior down and skips the rest, so
/// taking them costs a few vertices rather than a call, and it shades only a
/// record's quad, the first two triangles of every band class's pattern.
pub(crate) fn interior_run_draws(draws: &[RunDrawCall]) -> SmallVec<[PlannedRunDraw; 4]> {
    struct Stretch {
        key: ShapePipelineKey,
        start: u32,
        span: Option<PlannedRunDraw>,
    }
    let mut planned: SmallVec<[PlannedRunDraw; 4]> = SmallVec::new();
    let mut stretch: Option<Stretch> = None;
    for draw in draws.iter().rev() {
        let Some(key) = draw.key.interior() else {
            planned.extend(stretch.take().and_then(|open| open.span));
            continue;
        };
        if stretch
            .as_ref()
            .is_some_and(|open| open.key != key || open.start != draw.records.end)
        {
            planned.extend(stretch.take().and_then(|open| open.span));
        }
        let open = stretch.get_or_insert(Stretch {
            key,
            start: draw.records.end,
            span: None,
        });
        open.start = draw.records.start;
        if !draw.occluders {
            continue;
        }
        match &mut open.span {
            Some(span) => span.records.start = draw.records.start,
            None => {
                open.span = Some(PlannedRunDraw {
                    key,
                    band_class: draw.band_class,
                    indices: 0..cranpose_ui_graphics::strip_indices(1),
                    records: draw.records.clone(),
                });
            }
        }
    }
    planned.extend(stretch.and_then(|open| open.span));
    planned
}

/// Drops the entries of `cache` no frame drew for more than `idle_frames`
/// before `frame`: the least recently used first, which is the order they
/// were last drawn in.
fn evict_idle<K: Clone + Eq + std::hash::Hash, V>(
    cache: &mut BoundedLruCache<K, V>,
    frame: u64,
    idle_frames: u64,
    last_frame: impl Fn(&V) -> u64,
) {
    while cache
        .peek_lru()
        .is_some_and(|(_, value)| frame.saturating_sub(last_frame(value)) > idle_frames)
    {
        cache.pop_lru();
    }
}

/// Cuts the run draws from `start` on to `window` of their records, counted
/// across them in order, and drops the draws left with none.
fn window_draws(draws: &mut Vec<RunDrawCall>, start: usize, window: &std::ops::Range<u32>) {
    let mut relative = 0u32;
    let mut kept = start;
    for index in start..draws.len() {
        let draw = &mut draws[index];
        let count = draw.records.end - draw.records.start;
        let first = relative;
        relative += count;
        let keep_start = window.start.max(first).min(first + count);
        let keep_end = window.end.min(first + count).max(keep_start);
        draw.records =
            draw.records.start + (keep_start - first)..draw.records.start + (keep_end - first);
        if draw.records.start < draw.records.end {
            draws.swap(kept, index);
            kept += 1;
        }
    }
    draws.truncate(kept);
}

#[cfg(test)]
#[path = "tests/render_text_bounds_tests.rs"]
mod text_bounds_tests;

#[cfg(test)]
#[path = "tests/render_retained_glyph_tests.rs"]
mod retained_glyph_tests;

#[cfg(test)]
#[path = "tests/frame_clear_tests.rs"]
mod frame_clear_tests;

#[cfg(test)]
#[path = "tests/render_clip_scissor_tests.rs"]
mod clip_scissor_tests;

#[cfg(test)]
#[path = "tests/glyph_kind_tests.rs"]
mod glyph_kind_tests;

#[cfg(test)]
#[path = "tests/glyph_run_bounds_tests.rs"]
mod glyph_run_bounds_tests;

#[cfg(test)]
#[path = "tests/shape_variant_tests.rs"]
mod shape_variant_tests;