wgpu_core/command/
transfer.rs

1use alloc::{sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{
6    error::{ErrorType, WebGpuError},
7    BufferAddress, BufferTextureCopyInfoError, BufferUsages, Extent3d, TextureSelector,
8    TextureUsages,
9};
10
11#[cfg(feature = "trace")]
12use crate::device::trace::Command as TraceCommand;
13use crate::{
14    api_log,
15    command::{clear_texture, CommandEncoderError, EncoderStateError},
16    conv,
17    device::{Device, MissingDownlevelFlags},
18    global::Global,
19    id::{BufferId, CommandEncoderId, TextureId},
20    init_tracker::{
21        has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
22        TextureInitTrackerAction,
23    },
24    resource::{
25        MissingBufferUsageError, MissingTextureUsageError, ParentDevice, RawResourceAccess,
26        Texture, TextureErrorDimension,
27    },
28    snatch::SnatchGuard,
29};
30
31use super::{ClearError, CommandBufferMutable};
32
33pub type TexelCopyBufferInfo = wgt::TexelCopyBufferInfo<BufferId>;
34pub type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<TextureId>;
35pub type CopyExternalImageDestInfo = wgt::CopyExternalImageDestInfo<TextureId>;
36
37#[derive(Clone, Copy, Debug)]
38pub enum CopySide {
39    Source,
40    Destination,
41}
42
43/// Error encountered while attempting a data transfer.
44#[derive(Clone, Debug, Error)]
45#[non_exhaustive]
46pub enum TransferError {
47    #[error("Source and destination cannot be the same buffer")]
48    SameSourceDestinationBuffer,
49    #[error(transparent)]
50    MissingBufferUsage(#[from] MissingBufferUsageError),
51    #[error(transparent)]
52    MissingTextureUsage(#[from] MissingTextureUsageError),
53    #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
54    BufferOverrun {
55        start_offset: BufferAddress,
56        end_offset: BufferAddress,
57        buffer_size: BufferAddress,
58        side: CopySide,
59    },
60    #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
61    TextureOverrun {
62        start_offset: u32,
63        end_offset: u32,
64        texture_size: u32,
65        dimension: TextureErrorDimension,
66        side: CopySide,
67    },
68    #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
69    InvalidTextureAspect {
70        format: wgt::TextureFormat,
71        aspect: wgt::TextureAspect,
72    },
73    #[error("Unable to select texture mip level {level} out of {total}")]
74    InvalidTextureMipLevel { level: u32, total: u32 },
75    #[error("Texture dimension must be 2D when copying from an external texture")]
76    InvalidDimensionExternal,
77    #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
78    UnalignedBufferOffset(BufferAddress),
79    #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
80    UnalignedCopySize(BufferAddress),
81    #[error("Copy width is not a multiple of block width")]
82    UnalignedCopyWidth,
83    #[error("Copy height is not a multiple of block height")]
84    UnalignedCopyHeight,
85    #[error("Copy origin's x component is not a multiple of block width")]
86    UnalignedCopyOriginX,
87    #[error("Copy origin's y component is not a multiple of block height")]
88    UnalignedCopyOriginY,
89    #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
90    UnalignedBytesPerRow,
91    #[error("Number of bytes per row needs to be specified since more than one row is copied")]
92    UnspecifiedBytesPerRow,
93    #[error("Number of rows per image needs to be specified since more than one image is copied")]
94    UnspecifiedRowsPerImage,
95    #[error("Number of bytes per row is less than the number of bytes in a complete row")]
96    InvalidBytesPerRow,
97    #[error("Number of rows per image is invalid")]
98    InvalidRowsPerImage,
99    #[error("Overflow while computing the size of the copy")]
100    SizeOverflow,
101    #[error("Copy source aspects must refer to all aspects of the source texture format")]
102    CopySrcMissingAspects,
103    #[error(
104        "Copy destination aspects must refer to all aspects of the destination texture format"
105    )]
106    CopyDstMissingAspects,
107    #[error("Copy aspect must refer to a single aspect of texture format")]
108    CopyAspectNotOne,
109    #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
110    CopyFromForbiddenTextureFormat {
111        format: wgt::TextureFormat,
112        aspect: wgt::TextureAspect,
113    },
114    #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
115    CopyToForbiddenTextureFormat {
116        format: wgt::TextureFormat,
117        aspect: wgt::TextureAspect,
118    },
119    #[error(
120        "Copying to textures with format {0:?} is forbidden when copying from external texture"
121    )]
122    ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
123    #[error(
124        "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
125    )]
126    TextureFormatsNotCopyCompatible {
127        src_format: wgt::TextureFormat,
128        dst_format: wgt::TextureFormat,
129    },
130    #[error(transparent)]
131    MemoryInitFailure(#[from] ClearError),
132    #[error("Cannot encode this copy because of a missing downelevel flag")]
133    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
134    #[error("Source texture sample count must be 1, got {sample_count}")]
135    InvalidSampleCount { sample_count: u32 },
136    #[error(
137        "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
138    )]
139    SampleCountNotEqual {
140        src_sample_count: u32,
141        dst_sample_count: u32,
142    },
143    #[error("Requested mip level {requested} does no exist (count: {count})")]
144    InvalidMipLevel { requested: u32, count: u32 },
145}
146
147impl WebGpuError for TransferError {
148    fn webgpu_error_type(&self) -> ErrorType {
149        let e: &dyn WebGpuError = match self {
150            Self::MissingBufferUsage(e) => e,
151            Self::MissingTextureUsage(e) => e,
152            Self::MemoryInitFailure(e) => e,
153
154            Self::BufferOverrun { .. }
155            | Self::TextureOverrun { .. }
156            | Self::InvalidTextureAspect { .. }
157            | Self::InvalidTextureMipLevel { .. }
158            | Self::InvalidDimensionExternal
159            | Self::UnalignedBufferOffset(..)
160            | Self::UnalignedCopySize(..)
161            | Self::UnalignedCopyWidth
162            | Self::UnalignedCopyHeight
163            | Self::UnalignedCopyOriginX
164            | Self::UnalignedCopyOriginY
165            | Self::UnalignedBytesPerRow
166            | Self::UnspecifiedBytesPerRow
167            | Self::UnspecifiedRowsPerImage
168            | Self::InvalidBytesPerRow
169            | Self::InvalidRowsPerImage
170            | Self::SizeOverflow
171            | Self::CopySrcMissingAspects
172            | Self::CopyDstMissingAspects
173            | Self::CopyAspectNotOne
174            | Self::CopyFromForbiddenTextureFormat { .. }
175            | Self::CopyToForbiddenTextureFormat { .. }
176            | Self::ExternalCopyToForbiddenTextureFormat(..)
177            | Self::TextureFormatsNotCopyCompatible { .. }
178            | Self::MissingDownlevelFlags(..)
179            | Self::InvalidSampleCount { .. }
180            | Self::SampleCountNotEqual { .. }
181            | Self::InvalidMipLevel { .. }
182            | Self::SameSourceDestinationBuffer => return ErrorType::Validation,
183        };
184        e.webgpu_error_type()
185    }
186}
187
188impl From<BufferTextureCopyInfoError> for TransferError {
189    fn from(value: BufferTextureCopyInfoError) -> Self {
190        match value {
191            BufferTextureCopyInfoError::InvalidBytesPerRow => Self::InvalidBytesPerRow,
192            BufferTextureCopyInfoError::InvalidRowsPerImage => Self::InvalidRowsPerImage,
193            BufferTextureCopyInfoError::ImageStrideOverflow
194            | BufferTextureCopyInfoError::ImageBytesOverflow(_)
195            | BufferTextureCopyInfoError::ArraySizeOverflow(_) => Self::SizeOverflow,
196        }
197    }
198}
199
200pub(crate) fn extract_texture_selector<T>(
201    copy_texture: &wgt::TexelCopyTextureInfo<T>,
202    copy_size: &Extent3d,
203    texture: &Texture,
204) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
205    let format = texture.desc.format;
206    let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
207    if copy_aspect.is_empty() {
208        return Err(TransferError::InvalidTextureAspect {
209            format,
210            aspect: copy_texture.aspect,
211        });
212    }
213
214    let (layers, origin_z) = match texture.desc.dimension {
215        wgt::TextureDimension::D1 => (0..1, 0),
216        wgt::TextureDimension::D2 => (
217            copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
218            0,
219        ),
220        wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
221    };
222    let base = hal::TextureCopyBase {
223        origin: wgt::Origin3d {
224            x: copy_texture.origin.x,
225            y: copy_texture.origin.y,
226            z: origin_z,
227        },
228        // this value will be incremented per copied layer
229        array_layer: layers.start,
230        mip_level: copy_texture.mip_level,
231        aspect: copy_aspect,
232    };
233    let selector = TextureSelector {
234        mips: copy_texture.mip_level..copy_texture.mip_level + 1,
235        layers,
236    };
237
238    Ok((selector, base))
239}
240
241/// WebGPU's [validating linear texture data][vltd] algorithm.
242///
243/// Copied with some modifications from WebGPU standard.
244///
245/// If successful, returns a pair `(bytes, stride)`, where:
246/// - `bytes` is the number of buffer bytes required for this copy, and
247/// - `stride` number of bytes between array layers.
248///
249/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
250pub(crate) fn validate_linear_texture_data(
251    layout: &wgt::TexelCopyBufferLayout,
252    format: wgt::TextureFormat,
253    aspect: wgt::TextureAspect,
254    buffer_size: BufferAddress,
255    buffer_side: CopySide,
256    copy_size: &Extent3d,
257    need_copy_aligned_rows: bool,
258) -> Result<(BufferAddress, BufferAddress), TransferError> {
259    let wgt::BufferTextureCopyInfo {
260        copy_width,
261        copy_height,
262        depth_or_array_layers,
263
264        offset,
265
266        block_size_bytes,
267        block_width_texels,
268        block_height_texels,
269
270        width_blocks: _,
271        height_blocks,
272
273        row_bytes_dense: _,
274        row_stride_bytes,
275
276        image_stride_rows: _,
277        image_stride_bytes,
278
279        image_rows_dense: _,
280        image_bytes_dense: _,
281
282        bytes_in_copy,
283    } = layout.get_buffer_texture_copy_info(format, aspect, copy_size)?;
284
285    if copy_width % block_width_texels != 0 {
286        return Err(TransferError::UnalignedCopyWidth);
287    }
288    if copy_height % block_height_texels != 0 {
289        return Err(TransferError::UnalignedCopyHeight);
290    }
291
292    let requires_multiple_rows = depth_or_array_layers > 1 || height_blocks > 1;
293    let requires_multiple_images = depth_or_array_layers > 1;
294
295    // `get_buffer_texture_copy_info()` already proceeded with defaults if these
296    // were not specified, and ensured that the values satisfy the minima if
297    // they were, but now we enforce the WebGPU requirement that they be
298    // specified any time they apply.
299    if layout.bytes_per_row.is_none() && requires_multiple_rows {
300        return Err(TransferError::UnspecifiedBytesPerRow);
301    }
302
303    if layout.rows_per_image.is_none() && requires_multiple_images {
304        return Err(TransferError::UnspecifiedRowsPerImage);
305    };
306
307    if need_copy_aligned_rows {
308        let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
309
310        let mut offset_alignment = block_size_bytes;
311        if format.is_depth_stencil_format() {
312            offset_alignment = 4
313        }
314        if offset % offset_alignment != 0 {
315            return Err(TransferError::UnalignedBufferOffset(offset));
316        }
317
318        // The alignment of row_stride_bytes is only required if there are
319        // multiple rows
320        if requires_multiple_rows && row_stride_bytes % bytes_per_row_alignment != 0 {
321            return Err(TransferError::UnalignedBytesPerRow);
322        }
323    }
324
325    if offset + bytes_in_copy > buffer_size {
326        return Err(TransferError::BufferOverrun {
327            start_offset: offset,
328            end_offset: offset + bytes_in_copy,
329            buffer_size,
330            side: buffer_side,
331        });
332    }
333
334    Ok((bytes_in_copy, image_stride_bytes))
335}
336
337/// WebGPU's [validating texture copy range][vtcr] algorithm.
338///
339/// Copied with minor modifications from WebGPU standard.
340///
341/// Returns the HAL copy extent and the layer count.
342///
343/// [vtcr]: https://gpuweb.github.io/gpuweb/#validating-texture-copy-range
344pub(crate) fn validate_texture_copy_range<T>(
345    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
346    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
347    texture_side: CopySide,
348    copy_size: &Extent3d,
349) -> Result<(hal::CopyExtent, u32), TransferError> {
350    let (block_width, block_height) = desc.format.block_dimensions();
351
352    let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
353        TransferError::InvalidTextureMipLevel {
354            level: texture_copy_view.mip_level,
355            total: desc.mip_level_count,
356        },
357    )?;
358    // physical size can be larger than the virtual
359    let extent = extent_virtual.physical_size(desc.format);
360
361    /// Return `Ok` if a run `size` texels long starting at `start_offset` falls
362    /// entirely within `texture_size`. Otherwise, return an appropriate a`Err`.
363    fn check_dimension(
364        dimension: TextureErrorDimension,
365        side: CopySide,
366        start_offset: u32,
367        size: u32,
368        texture_size: u32,
369    ) -> Result<(), TransferError> {
370        // Avoid underflow in the subtraction by checking start_offset against
371        // texture_size first.
372        if start_offset <= texture_size && size <= texture_size - start_offset {
373            Ok(())
374        } else {
375            Err(TransferError::TextureOverrun {
376                start_offset,
377                end_offset: start_offset.wrapping_add(size),
378                texture_size,
379                dimension,
380                side,
381            })
382        }
383    }
384
385    check_dimension(
386        TextureErrorDimension::X,
387        texture_side,
388        texture_copy_view.origin.x,
389        copy_size.width,
390        extent.width,
391    )?;
392    check_dimension(
393        TextureErrorDimension::Y,
394        texture_side,
395        texture_copy_view.origin.y,
396        copy_size.height,
397        extent.height,
398    )?;
399    check_dimension(
400        TextureErrorDimension::Z,
401        texture_side,
402        texture_copy_view.origin.z,
403        copy_size.depth_or_array_layers,
404        extent.depth_or_array_layers,
405    )?;
406
407    if texture_copy_view.origin.x % block_width != 0 {
408        return Err(TransferError::UnalignedCopyOriginX);
409    }
410    if texture_copy_view.origin.y % block_height != 0 {
411        return Err(TransferError::UnalignedCopyOriginY);
412    }
413    if copy_size.width % block_width != 0 {
414        return Err(TransferError::UnalignedCopyWidth);
415    }
416    if copy_size.height % block_height != 0 {
417        return Err(TransferError::UnalignedCopyHeight);
418    }
419
420    let (depth, array_layer_count) = match desc.dimension {
421        wgt::TextureDimension::D1 => (1, 1),
422        wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
423        wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
424    };
425
426    let copy_extent = hal::CopyExtent {
427        width: copy_size.width,
428        height: copy_size.height,
429        depth,
430    };
431    Ok((copy_extent, array_layer_count))
432}
433
434fn handle_texture_init(
435    init_kind: MemoryInitKind,
436    cmd_buf_data: &mut CommandBufferMutable,
437    device: &Device,
438    copy_texture: &TexelCopyTextureInfo,
439    copy_size: &Extent3d,
440    texture: &Arc<Texture>,
441    snatch_guard: &SnatchGuard<'_>,
442) -> Result<(), ClearError> {
443    let init_action = TextureInitTrackerAction {
444        texture: texture.clone(),
445        range: TextureInitRange {
446            mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
447            layer_range: copy_texture.origin.z
448                ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
449        },
450        kind: init_kind,
451    };
452
453    // Register the init action.
454    let immediate_inits = cmd_buf_data
455        .texture_memory_actions
456        .register_init_action(&{ init_action });
457
458    // In rare cases we may need to insert an init operation immediately onto the command buffer.
459    if !immediate_inits.is_empty() {
460        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
461        for init in immediate_inits {
462            clear_texture(
463                &init.texture,
464                TextureInitRange {
465                    mip_range: init.mip_level..(init.mip_level + 1),
466                    layer_range: init.layer..(init.layer + 1),
467                },
468                cmd_buf_raw,
469                &mut cmd_buf_data.trackers.textures,
470                &device.alignments,
471                device.zero_buffer.as_ref(),
472                snatch_guard,
473            )?;
474        }
475    }
476
477    Ok(())
478}
479
480/// Prepare a transfer's source texture.
481///
482/// Ensure the source texture of a transfer is in the right initialization
483/// state, and record the state for after the transfer operation.
484fn handle_src_texture_init(
485    cmd_buf_data: &mut CommandBufferMutable,
486    device: &Device,
487    source: &TexelCopyTextureInfo,
488    copy_size: &Extent3d,
489    texture: &Arc<Texture>,
490    snatch_guard: &SnatchGuard<'_>,
491) -> Result<(), TransferError> {
492    handle_texture_init(
493        MemoryInitKind::NeedsInitializedMemory,
494        cmd_buf_data,
495        device,
496        source,
497        copy_size,
498        texture,
499        snatch_guard,
500    )?;
501    Ok(())
502}
503
504/// Prepare a transfer's destination texture.
505///
506/// Ensure the destination texture of a transfer is in the right initialization
507/// state, and record the state for after the transfer operation.
508fn handle_dst_texture_init(
509    cmd_buf_data: &mut CommandBufferMutable,
510    device: &Device,
511    destination: &TexelCopyTextureInfo,
512    copy_size: &Extent3d,
513    texture: &Arc<Texture>,
514    snatch_guard: &SnatchGuard<'_>,
515) -> Result<(), TransferError> {
516    // Attention: If we don't write full texture subresources, we need to a full
517    // clear first since we don't track subrects. This means that in rare cases
518    // even a *destination* texture of a transfer may need an immediate texture
519    // init.
520    let dst_init_kind = if has_copy_partial_init_tracker_coverage(
521        copy_size,
522        destination.mip_level,
523        &texture.desc,
524    ) {
525        MemoryInitKind::NeedsInitializedMemory
526    } else {
527        MemoryInitKind::ImplicitlyInitialized
528    };
529
530    handle_texture_init(
531        dst_init_kind,
532        cmd_buf_data,
533        device,
534        destination,
535        copy_size,
536        texture,
537        snatch_guard,
538    )?;
539    Ok(())
540}
541
542impl Global {
543    pub fn command_encoder_copy_buffer_to_buffer(
544        &self,
545        command_encoder_id: CommandEncoderId,
546        source: BufferId,
547        source_offset: BufferAddress,
548        destination: BufferId,
549        destination_offset: BufferAddress,
550        size: Option<BufferAddress>,
551    ) -> Result<(), EncoderStateError> {
552        profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
553        api_log!(
554            "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
555        );
556
557        let hub = &self.hub;
558
559        let cmd_buf = hub
560            .command_buffers
561            .get(command_encoder_id.into_command_buffer_id());
562        let mut cmd_buf_data = cmd_buf.data.lock();
563        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
564            let device = &cmd_buf.device;
565            device.check_is_valid()?;
566
567            if source == destination {
568                return Err(TransferError::SameSourceDestinationBuffer.into());
569            }
570
571            #[cfg(feature = "trace")]
572            if let Some(ref mut list) = cmd_buf_data.commands {
573                list.push(TraceCommand::CopyBufferToBuffer {
574                    src: source,
575                    src_offset: source_offset,
576                    dst: destination,
577                    dst_offset: destination_offset,
578                    size,
579                });
580            }
581
582            let snatch_guard = device.snatchable_lock.read();
583
584            let src_buffer = hub.buffers.get(source).get()?;
585
586            src_buffer.same_device_as(cmd_buf.as_ref())?;
587
588            let src_pending = cmd_buf_data
589                .trackers
590                .buffers
591                .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
592
593            let src_raw = src_buffer.try_raw(&snatch_guard)?;
594            src_buffer
595                .check_usage(BufferUsages::COPY_SRC)
596                .map_err(TransferError::MissingBufferUsage)?;
597            // expecting only a single barrier
598            let src_barrier =
599                src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
600
601            let dst_buffer = hub.buffers.get(destination).get()?;
602
603            dst_buffer.same_device_as(cmd_buf.as_ref())?;
604
605            let dst_pending = cmd_buf_data
606                .trackers
607                .buffers
608                .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
609
610            let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
611            dst_buffer
612                .check_usage(BufferUsages::COPY_DST)
613                .map_err(TransferError::MissingBufferUsage)?;
614            let dst_barrier =
615                dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
616
617            let (size, source_end_offset) = match size {
618                Some(size) => (size, source_offset + size),
619                None => (src_buffer.size - source_offset, src_buffer.size),
620            };
621
622            if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
623                return Err(TransferError::UnalignedCopySize(size).into());
624            }
625            if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
626                return Err(TransferError::UnalignedBufferOffset(source_offset).into());
627            }
628            if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
629                return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
630            }
631            if !device
632                .downlevel
633                .flags
634                .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
635                && (src_buffer.usage.contains(BufferUsages::INDEX)
636                    || dst_buffer.usage.contains(BufferUsages::INDEX))
637            {
638                let forbidden_usages = BufferUsages::VERTEX
639                    | BufferUsages::UNIFORM
640                    | BufferUsages::INDIRECT
641                    | BufferUsages::STORAGE;
642                if src_buffer.usage.intersects(forbidden_usages)
643                    || dst_buffer.usage.intersects(forbidden_usages)
644                {
645                    return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
646                        wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
647                    ))
648                    .into());
649                }
650            }
651
652            let destination_end_offset = destination_offset + size;
653            if source_end_offset > src_buffer.size {
654                return Err(TransferError::BufferOverrun {
655                    start_offset: source_offset,
656                    end_offset: source_end_offset,
657                    buffer_size: src_buffer.size,
658                    side: CopySide::Source,
659                }
660                .into());
661            }
662            if destination_end_offset > dst_buffer.size {
663                return Err(TransferError::BufferOverrun {
664                    start_offset: destination_offset,
665                    end_offset: destination_end_offset,
666                    buffer_size: dst_buffer.size,
667                    side: CopySide::Destination,
668                }
669                .into());
670            }
671
672            if size == 0 {
673                log::trace!("Ignoring copy_buffer_to_buffer of size 0");
674                return Ok(());
675            }
676
677            // Make sure source is initialized memory and mark dest as initialized.
678            cmd_buf_data.buffer_memory_init_actions.extend(
679                dst_buffer.initialization_status.read().create_action(
680                    &dst_buffer,
681                    destination_offset..(destination_offset + size),
682                    MemoryInitKind::ImplicitlyInitialized,
683                ),
684            );
685            cmd_buf_data.buffer_memory_init_actions.extend(
686                src_buffer.initialization_status.read().create_action(
687                    &src_buffer,
688                    source_offset..(source_offset + size),
689                    MemoryInitKind::NeedsInitializedMemory,
690                ),
691            );
692
693            let region = hal::BufferCopy {
694                src_offset: source_offset,
695                dst_offset: destination_offset,
696                size: wgt::BufferSize::new(size).unwrap(),
697            };
698            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
699            let barriers = src_barrier
700                .into_iter()
701                .chain(dst_barrier)
702                .collect::<Vec<_>>();
703            unsafe {
704                cmd_buf_raw.transition_buffers(&barriers);
705                cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
706            }
707
708            Ok(())
709        })
710    }
711
712    pub fn command_encoder_copy_buffer_to_texture(
713        &self,
714        command_encoder_id: CommandEncoderId,
715        source: &TexelCopyBufferInfo,
716        destination: &TexelCopyTextureInfo,
717        copy_size: &Extent3d,
718    ) -> Result<(), EncoderStateError> {
719        profiling::scope!("CommandEncoder::copy_buffer_to_texture");
720        api_log!(
721            "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
722            source.buffer,
723            destination.texture
724        );
725
726        let hub = &self.hub;
727
728        let cmd_buf = hub
729            .command_buffers
730            .get(command_encoder_id.into_command_buffer_id());
731        let mut cmd_buf_data = cmd_buf.data.lock();
732        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
733            let device = &cmd_buf.device;
734            device.check_is_valid()?;
735
736            #[cfg(feature = "trace")]
737            if let Some(ref mut list) = cmd_buf_data.commands {
738                list.push(TraceCommand::CopyBufferToTexture {
739                    src: *source,
740                    dst: *destination,
741                    size: *copy_size,
742                });
743            }
744
745            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
746            {
747                log::trace!("Ignoring copy_buffer_to_texture of size 0");
748                return Ok(());
749            }
750
751            let dst_texture = hub.textures.get(destination.texture).get()?;
752
753            dst_texture.same_device_as(cmd_buf.as_ref())?;
754
755            let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
756                destination,
757                &dst_texture.desc,
758                CopySide::Destination,
759                copy_size,
760            )?;
761
762            let (dst_range, dst_base) =
763                extract_texture_selector(destination, copy_size, &dst_texture)?;
764
765            let snatch_guard = device.snatchable_lock.read();
766
767            // Handle texture init *before* dealing with barrier transitions so we
768            // have an easier time inserting "immediate-inits" that may be required
769            // by prior discards in rare cases.
770            handle_dst_texture_init(
771                cmd_buf_data,
772                device,
773                destination,
774                copy_size,
775                &dst_texture,
776                &snatch_guard,
777            )?;
778
779            let src_buffer = hub.buffers.get(source.buffer).get()?;
780
781            src_buffer.same_device_as(cmd_buf.as_ref())?;
782
783            let src_pending = cmd_buf_data
784                .trackers
785                .buffers
786                .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
787
788            let src_raw = src_buffer.try_raw(&snatch_guard)?;
789            src_buffer
790                .check_usage(BufferUsages::COPY_SRC)
791                .map_err(TransferError::MissingBufferUsage)?;
792            let src_barrier =
793                src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
794
795            let dst_pending = cmd_buf_data.trackers.textures.set_single(
796                &dst_texture,
797                dst_range,
798                wgt::TextureUses::COPY_DST,
799            );
800            let dst_raw = dst_texture.try_raw(&snatch_guard)?;
801            dst_texture
802                .check_usage(TextureUsages::COPY_DST)
803                .map_err(TransferError::MissingTextureUsage)?;
804            let dst_barrier = dst_pending
805                .map(|pending| pending.into_hal(dst_raw))
806                .collect::<Vec<_>>();
807
808            if !dst_base.aspect.is_one() {
809                return Err(TransferError::CopyAspectNotOne.into());
810            }
811
812            if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format, destination.aspect)
813            {
814                return Err(TransferError::CopyToForbiddenTextureFormat {
815                    format: dst_texture.desc.format,
816                    aspect: destination.aspect,
817                }
818                .into());
819            }
820
821            let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
822                validate_linear_texture_data(
823                    &source.layout,
824                    dst_texture.desc.format,
825                    destination.aspect,
826                    src_buffer.size,
827                    CopySide::Source,
828                    copy_size,
829                    true,
830                )?;
831
832            if dst_texture.desc.format.is_depth_stencil_format() {
833                device
834                    .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
835                    .map_err(TransferError::from)?;
836            }
837
838            cmd_buf_data.buffer_memory_init_actions.extend(
839                src_buffer.initialization_status.read().create_action(
840                    &src_buffer,
841                    source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
842                    MemoryInitKind::NeedsInitializedMemory,
843                ),
844            );
845
846            let regions = (0..array_layer_count)
847                .map(|rel_array_layer| {
848                    let mut texture_base = dst_base.clone();
849                    texture_base.array_layer += rel_array_layer;
850                    let mut buffer_layout = source.layout;
851                    buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
852                    hal::BufferTextureCopy {
853                        buffer_layout,
854                        texture_base,
855                        size: hal_copy_size,
856                    }
857                })
858                .collect::<Vec<_>>();
859
860            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
861            unsafe {
862                cmd_buf_raw.transition_textures(&dst_barrier);
863                cmd_buf_raw.transition_buffers(src_barrier.as_slice());
864                cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, &regions);
865            }
866
867            Ok(())
868        })
869    }
870
871    pub fn command_encoder_copy_texture_to_buffer(
872        &self,
873        command_encoder_id: CommandEncoderId,
874        source: &TexelCopyTextureInfo,
875        destination: &TexelCopyBufferInfo,
876        copy_size: &Extent3d,
877    ) -> Result<(), EncoderStateError> {
878        profiling::scope!("CommandEncoder::copy_texture_to_buffer");
879        api_log!(
880            "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
881            source.texture,
882            destination.buffer
883        );
884
885        let hub = &self.hub;
886
887        let cmd_buf = hub
888            .command_buffers
889            .get(command_encoder_id.into_command_buffer_id());
890        let mut cmd_buf_data = cmd_buf.data.lock();
891        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
892            let device = &cmd_buf.device;
893            device.check_is_valid()?;
894
895            #[cfg(feature = "trace")]
896            if let Some(list) = cmd_buf_data.commands.as_mut() {
897                list.push(TraceCommand::CopyTextureToBuffer {
898                    src: *source,
899                    dst: *destination,
900                    size: *copy_size,
901                });
902            }
903
904            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
905            {
906                log::trace!("Ignoring copy_texture_to_buffer of size 0");
907                return Ok(());
908            }
909
910            let src_texture = hub.textures.get(source.texture).get()?;
911
912            src_texture.same_device_as(cmd_buf.as_ref())?;
913
914            let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
915                source,
916                &src_texture.desc,
917                CopySide::Source,
918                copy_size,
919            )?;
920
921            let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
922
923            let snatch_guard = device.snatchable_lock.read();
924
925            // Handle texture init *before* dealing with barrier transitions so we
926            // have an easier time inserting "immediate-inits" that may be required
927            // by prior discards in rare cases.
928            handle_src_texture_init(
929                cmd_buf_data,
930                device,
931                source,
932                copy_size,
933                &src_texture,
934                &snatch_guard,
935            )?;
936
937            let src_pending = cmd_buf_data.trackers.textures.set_single(
938                &src_texture,
939                src_range,
940                wgt::TextureUses::COPY_SRC,
941            );
942            let src_raw = src_texture.try_raw(&snatch_guard)?;
943            src_texture
944                .check_usage(TextureUsages::COPY_SRC)
945                .map_err(TransferError::MissingTextureUsage)?;
946            if src_texture.desc.sample_count != 1 {
947                return Err(TransferError::InvalidSampleCount {
948                    sample_count: src_texture.desc.sample_count,
949                }
950                .into());
951            }
952            if source.mip_level >= src_texture.desc.mip_level_count {
953                return Err(TransferError::InvalidMipLevel {
954                    requested: source.mip_level,
955                    count: src_texture.desc.mip_level_count,
956                }
957                .into());
958            }
959            let src_barrier = src_pending
960                .map(|pending| pending.into_hal(src_raw))
961                .collect::<Vec<_>>();
962
963            let dst_buffer = hub.buffers.get(destination.buffer).get()?;
964
965            dst_buffer.same_device_as(cmd_buf.as_ref())?;
966
967            let dst_pending = cmd_buf_data
968                .trackers
969                .buffers
970                .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
971
972            let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
973            dst_buffer
974                .check_usage(BufferUsages::COPY_DST)
975                .map_err(TransferError::MissingBufferUsage)?;
976            let dst_barrier =
977                dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
978
979            if !src_base.aspect.is_one() {
980                return Err(TransferError::CopyAspectNotOne.into());
981            }
982
983            if !conv::is_valid_copy_src_texture_format(src_texture.desc.format, source.aspect) {
984                return Err(TransferError::CopyFromForbiddenTextureFormat {
985                    format: src_texture.desc.format,
986                    aspect: source.aspect,
987                }
988                .into());
989            }
990
991            let (required_buffer_bytes_in_copy, bytes_per_array_layer) =
992                validate_linear_texture_data(
993                    &destination.layout,
994                    src_texture.desc.format,
995                    source.aspect,
996                    dst_buffer.size,
997                    CopySide::Destination,
998                    copy_size,
999                    true,
1000                )?;
1001
1002            if src_texture.desc.format.is_depth_stencil_format() {
1003                device
1004                    .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
1005                    .map_err(TransferError::from)?;
1006            }
1007
1008            cmd_buf_data.buffer_memory_init_actions.extend(
1009                dst_buffer.initialization_status.read().create_action(
1010                    &dst_buffer,
1011                    destination.layout.offset
1012                        ..(destination.layout.offset + required_buffer_bytes_in_copy),
1013                    MemoryInitKind::ImplicitlyInitialized,
1014                ),
1015            );
1016
1017            let regions = (0..array_layer_count)
1018                .map(|rel_array_layer| {
1019                    let mut texture_base = src_base.clone();
1020                    texture_base.array_layer += rel_array_layer;
1021                    let mut buffer_layout = destination.layout;
1022                    buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1023                    hal::BufferTextureCopy {
1024                        buffer_layout,
1025                        texture_base,
1026                        size: hal_copy_size,
1027                    }
1028                })
1029                .collect::<Vec<_>>();
1030            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1031            unsafe {
1032                cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
1033                cmd_buf_raw.transition_textures(&src_barrier);
1034                cmd_buf_raw.copy_texture_to_buffer(
1035                    src_raw,
1036                    wgt::TextureUses::COPY_SRC,
1037                    dst_raw,
1038                    &regions,
1039                );
1040            }
1041
1042            Ok(())
1043        })
1044    }
1045
1046    pub fn command_encoder_copy_texture_to_texture(
1047        &self,
1048        command_encoder_id: CommandEncoderId,
1049        source: &TexelCopyTextureInfo,
1050        destination: &TexelCopyTextureInfo,
1051        copy_size: &Extent3d,
1052    ) -> Result<(), EncoderStateError> {
1053        profiling::scope!("CommandEncoder::copy_texture_to_texture");
1054        api_log!(
1055            "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1056            source.texture,
1057            destination.texture
1058        );
1059
1060        let hub = &self.hub;
1061
1062        let cmd_buf = hub
1063            .command_buffers
1064            .get(command_encoder_id.into_command_buffer_id());
1065        let mut cmd_buf_data = cmd_buf.data.lock();
1066        cmd_buf_data.record_with(|cmd_buf_data| -> Result<(), CommandEncoderError> {
1067            let device = &cmd_buf.device;
1068            device.check_is_valid()?;
1069
1070            let snatch_guard = device.snatchable_lock.read();
1071
1072            #[cfg(feature = "trace")]
1073            if let Some(ref mut list) = cmd_buf_data.commands {
1074                list.push(TraceCommand::CopyTextureToTexture {
1075                    src: *source,
1076                    dst: *destination,
1077                    size: *copy_size,
1078                });
1079            }
1080
1081            if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0
1082            {
1083                log::trace!("Ignoring copy_texture_to_texture of size 0");
1084                return Ok(());
1085            }
1086
1087            let src_texture = hub.textures.get(source.texture).get()?;
1088            let dst_texture = hub.textures.get(destination.texture).get()?;
1089
1090            src_texture.same_device_as(cmd_buf.as_ref())?;
1091            dst_texture.same_device_as(cmd_buf.as_ref())?;
1092
1093            // src and dst texture format must be copy-compatible
1094            // https://gpuweb.github.io/gpuweb/#copy-compatible
1095            if src_texture.desc.format.remove_srgb_suffix()
1096                != dst_texture.desc.format.remove_srgb_suffix()
1097            {
1098                return Err(TransferError::TextureFormatsNotCopyCompatible {
1099                    src_format: src_texture.desc.format,
1100                    dst_format: dst_texture.desc.format,
1101                }
1102                .into());
1103            }
1104
1105            let (src_copy_size, array_layer_count) = validate_texture_copy_range(
1106                source,
1107                &src_texture.desc,
1108                CopySide::Source,
1109                copy_size,
1110            )?;
1111            let (dst_copy_size, _) = validate_texture_copy_range(
1112                destination,
1113                &dst_texture.desc,
1114                CopySide::Destination,
1115                copy_size,
1116            )?;
1117
1118            let (src_range, src_tex_base) =
1119                extract_texture_selector(source, copy_size, &src_texture)?;
1120            let (dst_range, dst_tex_base) =
1121                extract_texture_selector(destination, copy_size, &dst_texture)?;
1122            let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1123            let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1124            if src_tex_base.aspect != src_texture_aspects {
1125                return Err(TransferError::CopySrcMissingAspects.into());
1126            }
1127            if dst_tex_base.aspect != dst_texture_aspects {
1128                return Err(TransferError::CopyDstMissingAspects.into());
1129            }
1130
1131            if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1132                return Err(TransferError::SampleCountNotEqual {
1133                    src_sample_count: src_texture.desc.sample_count,
1134                    dst_sample_count: dst_texture.desc.sample_count,
1135                }
1136                .into());
1137            }
1138
1139            // Handle texture init *before* dealing with barrier transitions so we
1140            // have an easier time inserting "immediate-inits" that may be required
1141            // by prior discards in rare cases.
1142            handle_src_texture_init(
1143                cmd_buf_data,
1144                device,
1145                source,
1146                copy_size,
1147                &src_texture,
1148                &snatch_guard,
1149            )?;
1150            handle_dst_texture_init(
1151                cmd_buf_data,
1152                device,
1153                destination,
1154                copy_size,
1155                &dst_texture,
1156                &snatch_guard,
1157            )?;
1158
1159            let src_pending = cmd_buf_data.trackers.textures.set_single(
1160                &src_texture,
1161                src_range,
1162                wgt::TextureUses::COPY_SRC,
1163            );
1164            let src_raw = src_texture.try_raw(&snatch_guard)?;
1165            src_texture
1166                .check_usage(TextureUsages::COPY_SRC)
1167                .map_err(TransferError::MissingTextureUsage)?;
1168
1169            //TODO: try to avoid this the collection. It's needed because both
1170            // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
1171            let mut barriers: ArrayVec<_, 2> = src_pending
1172                .map(|pending| pending.into_hal(src_raw))
1173                .collect();
1174
1175            let dst_pending = cmd_buf_data.trackers.textures.set_single(
1176                &dst_texture,
1177                dst_range,
1178                wgt::TextureUses::COPY_DST,
1179            );
1180            let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1181            dst_texture
1182                .check_usage(TextureUsages::COPY_DST)
1183                .map_err(TransferError::MissingTextureUsage)?;
1184
1185            barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1186
1187            let hal_copy_size = hal::CopyExtent {
1188                width: src_copy_size.width.min(dst_copy_size.width),
1189                height: src_copy_size.height.min(dst_copy_size.height),
1190                depth: src_copy_size.depth.min(dst_copy_size.depth),
1191            };
1192            let regions = (0..array_layer_count)
1193                .map(|rel_array_layer| {
1194                    let mut src_base = src_tex_base.clone();
1195                    let mut dst_base = dst_tex_base.clone();
1196                    src_base.array_layer += rel_array_layer;
1197                    dst_base.array_layer += rel_array_layer;
1198                    hal::TextureCopy {
1199                        src_base,
1200                        dst_base,
1201                        size: hal_copy_size,
1202                    }
1203                })
1204                .collect::<Vec<_>>();
1205            let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1206            unsafe {
1207                cmd_buf_raw.transition_textures(&barriers);
1208                cmd_buf_raw.copy_texture_to_texture(
1209                    src_raw,
1210                    wgt::TextureUses::COPY_SRC,
1211                    dst_raw,
1212                    &regions,
1213                );
1214            }
1215
1216            Ok(())
1217        })
1218    }
1219}
wgpu_core/command/transfer.rs

wgpu_core/command/
transfer.rs
