Skip to main content

wgpu_core/device/
resource.rs

1use alloc::{
2    borrow::Cow,
3    boxed::Box,
4    string::{String, ToString as _},
5    sync::{Arc, Weak},
6    vec::Vec,
7};
8use core::{
9    fmt,
10    mem::{self, ManuallyDrop},
11    num::NonZeroU32,
12    sync::atomic::{AtomicBool, Ordering},
13};
14use hal::ShouldBeNonZeroExt;
15
16use arrayvec::ArrayVec;
17use bitflags::Flags;
18use smallvec::SmallVec;
19use wgt::{
20    math::align_to, DeviceLostReason, TextureFormat, TextureSampleType, TextureSelector,
21    TextureViewDimension,
22};
23
24#[cfg(feature = "trace")]
25use crate::device::trace;
26use crate::{
27    api_log,
28    binding_model::{
29        self, BindGroup, BindGroupLateBufferBindingInfo, BindGroupLayout,
30        BindGroupLayoutEntryError, BindGroupLayoutState, CreateBindGroupError,
31        CreateBindGroupLayoutError,
32    },
33    command, conv,
34    device::{
35        bgl, create_validator, features_to_naga_capabilities, life::WaitIdleError, map_buffer,
36        AttachmentData, DeviceLostInvocation, HostMap, MissingDownlevelFlags, MissingFeatures,
37        RenderPassContext,
38    },
39    hal_label,
40    init_tracker::{
41        BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
42        TextureInitTrackerAction,
43    },
44    instance::{Adapter, RequestDeviceError},
45    lock::{rank, Mutex, RwLock},
46    pipeline::{self, ColorStateError},
47    pool::ResourcePool,
48    present,
49    resource::{
50        self, Buffer, ExternalTexture, Fallible, Labeled, ParentDevice, QuerySet,
51        RawResourceAccess, ResourceState, Sampler, StagingBuffer, Texture, TextureView,
52        TextureViewNotRenderableReason, Tlas, TrackingData,
53    },
54    resource_log,
55    snatch::{SnatchGuard, SnatchLock, Snatchable},
56    timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
57    track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
58    validation::{self, check_color_attachment_count, PassthroughInterface, ShaderMetaData},
59    weak_vec::WeakVec,
60    FastHashMap, LabelHelpers, OnceCellOrLock,
61};
62
63use super::{
64    queue::Queue, surface_config::validate_surface_configuration, DeviceDescriptor, DeviceError,
65    DeviceLostClosure, UserClosures, ENTRYPOINT_FAILURE_ERROR, ZERO_BUFFER_SIZE,
66};
67
68#[cfg(supports_64bit_atomics)]
69use core::sync::atomic::AtomicU64;
70#[cfg(not(supports_64bit_atomics))]
71use portable_atomic::AtomicU64;
72
73pub(crate) struct CommandIndices {
74    /// The index of the last command submission that was attempted.
75    ///
76    /// Note that `fence` may never be signalled with this value, if the command
77    /// submission failed. If you need to wait for everything running on a
78    /// `Queue` to complete, wait for [`last_successful_submission_index`].
79    ///
80    /// [`last_successful_submission_index`]: Device::last_successful_submission_index
81    pub(crate) active_submission_index: hal::FenceValue,
82    pub(crate) next_acceleration_structure_build_command_index: u64,
83}
84
85/// Parameters provided to shaders via a uniform buffer of the type
86/// [`NagaExternalTextureParams`], describing an [`ExternalTexture`] resource
87/// binding.
88///
89/// [`NagaExternalTextureParams`]: naga::SpecialTypes::external_texture_params
90/// [`ExternalTexture`]: binding_model::BindingResource::ExternalTexture
91#[repr(C)]
92#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
93pub struct ExternalTextureParams {
94    /// 4x4 column-major matrix with which to convert sampled YCbCr values
95    /// to RGBA.
96    ///
97    /// This is ignored when `num_planes` is 1.
98    pub yuv_conversion_matrix: [f32; 16],
99
100    /// 3x3 column-major matrix to transform linear RGB values in the source
101    /// color space to linear RGB values in the destination color space. In
102    /// combination with [`Self::src_transfer_function`] and
103    /// [`Self::dst_transfer_function`] this can be used to ensure that
104    /// [`ImageSample`] and [`ImageLoad`] operations return values in the
105    /// desired destination color space rather than the source color space of
106    /// the underlying planes.
107    ///
108    /// Includes a padding element after each column.
109    ///
110    /// [`ImageSample`]: naga::ir::Expression::ImageSample
111    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
112    pub gamut_conversion_matrix: [f32; 12],
113
114    /// Transfer function for the source color space. The *inverse* of this
115    /// will be applied to decode non-linear RGB to linear RGB in the source
116    /// color space.
117    pub src_transfer_function: wgt::ExternalTextureTransferFunction,
118
119    /// Transfer function for the destination color space. This will be applied
120    /// to encode linear RGB to non-linear RGB in the destination color space.
121    pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
122
123    /// Transform to apply to [`ImageSample`] coordinates.
124    ///
125    /// This is a 3x2 column-major matrix representing an affine transform from
126    /// normalized texture coordinates to the normalized coordinates that should
127    /// be sampled from the external texture's underlying plane(s).
128    ///
129    /// This transform may scale, translate, flip, and rotate in 90-degree
130    /// increments, but the result of transforming the rectangle (0,0)..(1,1)
131    /// must be an axis-aligned rectangle that falls within the bounds of
132    /// (0,0)..(1,1).
133    ///
134    /// [`ImageSample`]: naga::ir::Expression::ImageSample
135    pub sample_transform: [f32; 6],
136
137    /// Transform to apply to [`ImageLoad`] coordinates.
138    ///
139    /// This is a 3x2 column-major matrix representing an affine transform from
140    /// non-normalized texel coordinates to the non-normalized coordinates of
141    /// the texel that should be loaded from the external texture's underlying
142    /// plane 0. For planes 1 and 2, if present, plane 0's coordinates are
143    /// scaled according to the textures' relative sizes.
144    ///
145    /// This transform may scale, translate, flip, and rotate in 90-degree
146    /// increments, but the result of transforming the rectangle (0,0)..[`size`]
147    /// must be an axis-aligned rectangle that falls within the bounds of
148    /// (0,0)..[`size`].
149    ///
150    /// [`ImageLoad`]: naga::ir::Expression::ImageLoad
151    /// [`size`]: Self::size
152    pub load_transform: [f32; 6],
153
154    /// Size of the external texture.
155    ///
156    /// This is the value that should be returned by size queries in shader
157    /// code; it does not necessarily match the dimensions of the underlying
158    /// texture(s). As a special case, if this is `[0, 0]`, the actual size of
159    /// plane 0 should be used instead.
160    ///
161    /// This must be consistent with [`sample_transform`]: it should be the size
162    /// in texels of the rectangle covered by the square (0,0)..(1,1) after
163    /// [`sample_transform`] has been applied to it.
164    ///
165    /// [`sample_transform`]: Self::sample_transform
166    pub size: [u32; 2],
167
168    /// Number of planes. 1 indicates a single RGBA plane. 2 indicates a Y
169    /// plane and an interleaved CbCr plane. 3 indicates separate Y, Cb, and Cr
170    /// planes.
171    pub num_planes: u32,
172    // Ensure the size of this struct matches the type generated by Naga.
173    pub _padding: [u8; 4],
174}
175
176impl ExternalTextureParams {
177    pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
178        let gamut_conversion_matrix = [
179            desc.gamut_conversion_matrix[0],
180            desc.gamut_conversion_matrix[1],
181            desc.gamut_conversion_matrix[2],
182            0.0, // padding
183            desc.gamut_conversion_matrix[3],
184            desc.gamut_conversion_matrix[4],
185            desc.gamut_conversion_matrix[5],
186            0.0, // padding
187            desc.gamut_conversion_matrix[6],
188            desc.gamut_conversion_matrix[7],
189            desc.gamut_conversion_matrix[8],
190            0.0, // padding
191        ];
192
193        Self {
194            yuv_conversion_matrix: desc.yuv_conversion_matrix,
195            gamut_conversion_matrix,
196            src_transfer_function: desc.src_transfer_function,
197            dst_transfer_function: desc.dst_transfer_function,
198            size: [desc.width, desc.height],
199            sample_transform: desc.sample_transform,
200            load_transform: desc.load_transform,
201            num_planes: desc.num_planes() as u32,
202            _padding: Default::default(),
203        }
204    }
205}
206
207/// Structure describing a logical device. Some members are internally mutable,
208/// stored behind mutexes.
209pub struct Device {
210    raw: Box<dyn hal::DynDevice>,
211    pub(crate) adapter: Arc<Adapter>,
212    pub(crate) queue: OnceCellOrLock<Weak<Queue>>,
213    pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
214    pub(crate) empty_bgl: ManuallyDrop<Box<dyn hal::DynBindGroupLayout>>,
215    /// The `label` from the descriptor used to create the resource.
216    label: String,
217
218    pub(crate) command_allocator: command::CommandAllocator,
219
220    pub(crate) command_indices: RwLock<CommandIndices>,
221
222    /// The index of the last successful submission to this device's
223    /// [`hal::Queue`].
224    ///
225    /// Unlike [`active_submission_index`], which is incremented each time
226    /// submission is attempted, this is updated only when submission succeeds,
227    /// so waiting for this value won't hang waiting for work that was never
228    /// submitted.
229    ///
230    /// [`active_submission_index`]: CommandIndices::active_submission_index
231    pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
232
233    pub(crate) fence: ManuallyDrop<Box<dyn hal::DynFence>>,
234    pub(crate) snatchable_lock: SnatchLock,
235
236    /// Is this device valid? Valid is closely associated with "lose the device",
237    /// which can be triggered by various methods, including at the end of device
238    /// destroy, and by any GPU errors that cause us to no longer trust the state
239    /// of the device. Ideally we would like to fold valid into the storage of
240    /// the device itself (for example as an Error enum), but unfortunately we
241    /// need to continue to be able to retrieve the device in poll_devices to
242    /// determine if it can be dropped. If our internal accesses of devices were
243    /// done through ref-counted references and external accesses checked for
244    /// Error enums, we wouldn't need this. For now, we need it. All the call
245    /// sites where we check it are areas that should be revisited if we start
246    /// using ref-counted references for internal access.
247    pub(crate) valid: AtomicBool,
248
249    /// Closure to be called on "lose the device". This is invoked directly by
250    /// device.lose or by the UserCallbacks returned from maintain when the device
251    /// has been destroyed and its queues are empty.
252    pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
253
254    /// Stores the state of buffers and textures.
255    pub(crate) trackers: Mutex<DeviceTracker>,
256    pub(crate) tracker_indices: TrackerIndexAllocators,
257    /// Pool of bind group layouts, allowing deduplication.
258    pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
259    pub(crate) alignments: hal::Alignments,
260    pub(crate) limits: wgt::Limits,
261    pub(crate) features: wgt::Features,
262    pub(crate) downlevel: wgt::DownlevelCapabilities,
263    /// Buffer uses listed here, are expected to be ordered by the underlying hardware.
264    /// If a usage is ordered, then if the buffer state doesn't change between draw calls,
265    /// there are no barriers needed for synchronization.
266    /// See the implementations of [`hal::Adapter::get_ordered_buffer_usages`] for hardware specific info
267    pub(crate) ordered_buffer_usages: wgt::BufferUses,
268    /// Texture uses listed here, are expected to be ordered by the underlying hardware.
269    /// If a usage is ordered, then if the buffer state doesn't change between draw calls,
270    /// there are no barriers needed for synchronization.
271    /// See the implementations of [`hal::Adapter::get_ordered_texture_usages`] for hardware specific info
272    pub(crate) ordered_texture_usages: wgt::TextureUses,
273    pub(crate) instance_flags: wgt::InstanceFlags,
274    pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
275    pub(crate) usage_scopes: UsageScopePool,
276    pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
277    // Optional so that we can late-initialize this after the queue is created.
278    pub(crate) timestamp_normalizer:
279        OnceCellOrLock<crate::timestamp_normalization::TimestampNormalizer>,
280    /// Uniform buffer containing [`ExternalTextureParams`] with values such
281    /// that a [`TextureView`] bound to a [`wgt::BindingType::ExternalTexture`]
282    /// binding point will be rendered correctly. Intended to be used as the
283    /// [`hal::ExternalTextureBinding::params`] field.
284    pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
285    // needs to be dropped last
286    #[cfg(feature = "trace")]
287    pub(crate) trace: Mutex<Option<Box<dyn trace::Trace + Send + Sync + 'static>>>,
288}
289
290pub(crate) enum DeferredDestroy {
291    TextureViews(WeakVec<TextureView>),
292    BindGroups(WeakVec<BindGroup>),
293}
294
295impl fmt::Debug for Device {
296    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
297        f.debug_struct("Device")
298            .field("label", &self.label())
299            .field("limits", &self.limits)
300            .field("features", &self.features)
301            .field("downlevel", &self.downlevel)
302            .finish()
303    }
304}
305
306impl Drop for Device {
307    fn drop(&mut self) {
308        resource_log!("Drop {}", self.error_ident());
309
310        // SAFETY: We are in the Drop impl and we don't use self.zero_buffer anymore after this
311        // point.
312        let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
313        // SAFETY: We are in the Drop impl and we don't use self.empty_bgl anymore after this point.
314        let empty_bgl = unsafe { ManuallyDrop::take(&mut self.empty_bgl) };
315        // SAFETY: We are in the Drop impl and we don't use
316        // self.default_external_texture_params_buffer anymore after this point.
317        let default_external_texture_params_buffer =
318            unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
319        // SAFETY: We are in the Drop impl and we don't use self.fence anymore after this point.
320        let fence = unsafe { ManuallyDrop::take(&mut self.fence) };
321        if let Some(indirect_validation) = self.indirect_validation.take() {
322            indirect_validation.dispose(self.raw.as_ref());
323        }
324        if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
325            timestamp_normalizer.dispose(self.raw.as_ref());
326        }
327        unsafe {
328            self.raw.destroy_buffer(zero_buffer);
329            self.raw.destroy_bind_group_layout(empty_bgl);
330            self.raw
331                .destroy_buffer(default_external_texture_params_buffer);
332            self.raw.destroy_fence(fence);
333        }
334    }
335}
336
337impl Device {
338    pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
339        self.raw.as_ref()
340    }
341    pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
342        if self.features.contains(feature) {
343            Ok(())
344        } else {
345            Err(MissingFeatures(feature))
346        }
347    }
348
349    pub(crate) fn require_downlevel_flags(
350        &self,
351        flags: wgt::DownlevelFlags,
352    ) -> Result<(), MissingDownlevelFlags> {
353        if self.downlevel.flags.contains(flags) {
354            Ok(())
355        } else {
356            Err(MissingDownlevelFlags(flags))
357        }
358    }
359
360    /// # Safety
361    ///
362    /// - See [wgpu::Device::start_graphics_debugger_capture][api] for details the safety.
363    ///
364    /// [api]: ../../wgpu/struct.Device.html#method.start_graphics_debugger_capture
365    pub unsafe fn start_graphics_debugger_capture(&self) {
366        api_log!("Device::start_graphics_debugger_capture");
367
368        if !self.is_valid() {
369            return;
370        }
371        unsafe { self.raw().start_graphics_debugger_capture() };
372    }
373
374    /// # Safety
375    ///
376    /// - See [wgpu::Device::stop_graphics_debugger_capture][api] for details the safety.
377    ///
378    /// [api]: ../../wgpu/struct.Device.html#method.stop_graphics_debugger_capture
379    pub unsafe fn stop_graphics_debugger_capture(&self) {
380        api_log!("Device::stop_graphics_debugger_capture");
381
382        if !self.is_valid() {
383            return;
384        }
385        unsafe { self.raw().stop_graphics_debugger_capture() };
386    }
387}
388
389impl Device {
390    pub(crate) fn new(
391        raw_device: Box<dyn hal::DynDevice>,
392        adapter: &Arc<Adapter>,
393        desc: &DeviceDescriptor,
394        instance_flags: wgt::InstanceFlags,
395    ) -> Result<Self, DeviceError> {
396        #[cfg(not(feature = "trace"))]
397        match &desc.trace {
398            wgt::Trace::Off => {}
399            _ => {
400                log::error!("wgpu-core feature 'trace' is not enabled");
401            }
402        };
403        #[cfg(feature = "trace")]
404        let trace: Option<Box<dyn trace::Trace + Send + Sync + 'static>> = match &desc.trace {
405            wgt::Trace::Off => None,
406            wgt::Trace::Directory(dir) => match trace::DiskTrace::new(dir.clone()) {
407                Ok(mut trace) => {
408                    trace::Trace::add(
409                        &mut trace,
410                        trace::Action::Init {
411                            desc: wgt::DeviceDescriptor {
412                                trace: wgt::Trace::Off,
413                                ..desc.clone()
414                            },
415                            backend: adapter.backend(),
416                        },
417                    );
418                    Some(Box::new(trace))
419                }
420                Err(e) => {
421                    log::error!("Unable to start a trace in '{dir:?}': {e}");
422                    None
423                }
424            },
425            wgt::Trace::Memory => {
426                let mut trace = trace::MemoryTrace::new();
427                trace::Trace::add(
428                    &mut trace,
429                    trace::Action::Init {
430                        desc: wgt::DeviceDescriptor {
431                            trace: wgt::Trace::Off,
432                            ..desc.clone()
433                        },
434                        backend: adapter.backend(),
435                    },
436                );
437                Some(Box::new(trace))
438            }
439            // The enum is non_exhaustive, so we must have a fallback arm (that should be
440            // unreachable in practice).
441            t => {
442                log::error!("unimplemented wgpu_types::Trace variant {t:?}");
443                None
444            }
445        };
446
447        let ordered_buffer_usages = adapter.raw.adapter.get_ordered_buffer_usages();
448        let ordered_texture_usages = adapter.raw.adapter.get_ordered_texture_usages();
449
450        let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
451
452        let command_allocator = command::CommandAllocator::new();
453
454        let rt_uses = if desc
455            .required_features
456            .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
457        {
458            wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
459        } else {
460            wgt::BufferUses::empty()
461        };
462
463        // Create zeroed buffer used for texture clears (and raytracing if required).
464        let zero_buffer = unsafe {
465            raw_device.create_buffer(&hal::BufferDescriptor {
466                label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
467                size: ZERO_BUFFER_SIZE,
468                usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
469                memory_flags: hal::MemoryFlags::empty(),
470            })
471        }
472        .map_err(DeviceError::from_hal)?;
473
474        let empty_bgl = unsafe {
475            raw_device.create_bind_group_layout(&hal::BindGroupLayoutDescriptor {
476                label: None,
477                flags: hal::BindGroupLayoutFlags::empty(),
478                entries: &[],
479            })
480        }
481        .map_err(DeviceError::from_hal)?;
482
483        let default_external_texture_params_buffer = unsafe {
484            raw_device.create_buffer(&hal::BufferDescriptor {
485                label: hal_label(
486                    Some("(wgpu internal) default external texture params buffer"),
487                    instance_flags,
488                ),
489                size: size_of::<ExternalTextureParams>() as _,
490                usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
491                memory_flags: hal::MemoryFlags::empty(),
492            })
493        }
494        .map_err(DeviceError::from_hal)?;
495
496        // Cloned as we need them below anyway.
497        let alignments = adapter.raw.capabilities.alignments.clone();
498        let downlevel = adapter.raw.capabilities.downlevel.clone();
499        let limits = &adapter.raw.capabilities.limits;
500
501        let enable_indirect_validation = instance_flags
502            .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
503            && downlevel.flags.contains(
504                wgt::DownlevelFlags::INDIRECT_EXECUTION | wgt::DownlevelFlags::COMPUTE_SHADERS,
505            )
506            && limits.max_storage_buffers_per_shader_stage >= 2;
507
508        let indirect_validation = if enable_indirect_validation {
509            Some(crate::indirect_validation::IndirectValidation::new(
510                raw_device.as_ref(),
511                &desc.required_limits,
512                &desc.required_features,
513                instance_flags,
514                adapter.backend(),
515            )?)
516        } else {
517            None
518        };
519
520        Ok(Self {
521            raw: raw_device,
522            adapter: adapter.clone(),
523            queue: OnceCellOrLock::new(),
524            zero_buffer: ManuallyDrop::new(zero_buffer),
525            empty_bgl: ManuallyDrop::new(empty_bgl),
526            default_external_texture_params_buffer: ManuallyDrop::new(
527                default_external_texture_params_buffer,
528            ),
529            label: desc.label.to_string(),
530            command_allocator,
531            command_indices: RwLock::new(
532                rank::DEVICE_COMMAND_INDICES,
533                CommandIndices {
534                    active_submission_index: 0,
535                    // By starting at one, we can put the result in a NonZeroU64.
536                    next_acceleration_structure_build_command_index: 1,
537                },
538            ),
539            last_successful_submission_index: AtomicU64::new(0),
540            fence: ManuallyDrop::new(fence),
541            snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
542            valid: AtomicBool::new(true),
543            device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
544            trackers: Mutex::new(
545                rank::DEVICE_TRACKERS,
546                DeviceTracker::new(ordered_buffer_usages, ordered_texture_usages),
547            ),
548            tracker_indices: TrackerIndexAllocators::new(),
549            bgl_pool: ResourcePool::new(),
550            #[cfg(feature = "trace")]
551            trace: Mutex::new(rank::DEVICE_TRACE, trace),
552            alignments,
553            limits: desc.required_limits.clone(),
554            features: desc.required_features,
555            downlevel,
556            ordered_buffer_usages,
557            ordered_texture_usages,
558            instance_flags,
559            deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
560            usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
561            timestamp_normalizer: OnceCellOrLock::new(),
562            indirect_validation,
563        })
564    }
565
566    /// Initializes [`Device::default_external_texture_params_buffer`] with
567    /// required values such that a [`TextureView`] bound to a
568    /// [`wgt::BindingType::ExternalTexture`] binding point will be rendered
569    /// correctly.
570    fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
571        let data = ExternalTextureParams {
572            #[rustfmt::skip]
573            yuv_conversion_matrix: [
574                1.0, 0.0, 0.0, 0.0,
575                0.0, 1.0, 0.0, 0.0,
576                0.0, 0.0, 1.0, 0.0,
577                0.0, 0.0, 0.0, 1.0,
578            ],
579            #[rustfmt::skip]
580            gamut_conversion_matrix: [
581                1.0, 0.0, 0.0, /* padding */ 0.0,
582                0.0, 1.0, 0.0, /* padding */ 0.0,
583                0.0, 0.0, 1.0, /* padding */ 0.0,
584            ],
585            src_transfer_function: Default::default(),
586            dst_transfer_function: Default::default(),
587            size: [0, 0],
588            #[rustfmt::skip]
589            sample_transform: [
590                1.0, 0.0,
591                0.0, 1.0,
592                0.0, 0.0
593            ],
594            #[rustfmt::skip]
595            load_transform: [
596                1.0, 0.0,
597                0.0, 1.0,
598                0.0, 0.0
599            ],
600            num_planes: 1,
601            _padding: Default::default(),
602        };
603        let mut staging_buffer =
604            StagingBuffer::new(self, wgt::BufferSize::new(size_of_val(&data) as _).unwrap())?;
605        staging_buffer.write(bytemuck::bytes_of(&data));
606        let staging_buffer = staging_buffer.flush();
607
608        let params_buffer = self.default_external_texture_params_buffer.as_ref();
609        let queue = self.get_queue().unwrap();
610        let mut pending_writes = queue.pending_writes.lock();
611
612        unsafe {
613            pending_writes
614                .command_encoder
615                .transition_buffers(&[hal::BufferBarrier {
616                    buffer: params_buffer,
617                    usage: hal::StateTransition {
618                        from: wgt::BufferUses::MAP_WRITE,
619                        to: wgt::BufferUses::COPY_DST,
620                    },
621                }]);
622            pending_writes.command_encoder.copy_buffer_to_buffer(
623                staging_buffer.raw(),
624                params_buffer,
625                &[hal::BufferCopy {
626                    src_offset: 0,
627                    dst_offset: 0,
628                    size: staging_buffer.size,
629                }],
630            );
631            pending_writes.consume(staging_buffer);
632            pending_writes
633                .command_encoder
634                .transition_buffers(&[hal::BufferBarrier {
635                    buffer: params_buffer,
636                    usage: hal::StateTransition {
637                        from: wgt::BufferUses::COPY_DST,
638                        to: wgt::BufferUses::UNIFORM,
639                    },
640                }]);
641        }
642
643        Ok(())
644    }
645
646    pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
647        let queue = self.get_queue().unwrap();
648
649        let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
650            self,
651            queue.get_timestamp_period(),
652        )?;
653
654        self.timestamp_normalizer
655            .set(timestamp_normalizer)
656            .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
657
658        self.init_default_external_texture_params_buffer()?;
659
660        Ok(())
661    }
662
663    /// Returns the backend this device is using.
664    pub fn backend(&self) -> wgt::Backend {
665        self.adapter.backend()
666    }
667
668    pub fn is_valid(&self) -> bool {
669        self.valid.load(Ordering::Acquire)
670    }
671
672    pub fn check_is_valid(&self) -> Result<(), DeviceError> {
673        if self.is_valid() {
674            Ok(())
675        } else {
676            Err(DeviceError::Lost)
677        }
678    }
679
680    /// Stop tracing and return the trace object.
681    ///
682    /// This is mostly useful for in-memory traces.
683    #[cfg(feature = "trace")]
684    pub fn take_trace(&self) -> Option<Box<dyn trace::Trace + Send + Sync + 'static>> {
685        self.trace.lock().take()
686    }
687
688    /// Checks that we are operating within the memory budget reported by the native APIs.
689    ///
690    /// If we are not, the device gets invalidated.
691    ///
692    /// The budget might fluctuate over the lifetime of the application, so it should be checked
693    /// somewhat frequently.
694    pub fn lose_if_oom(&self) {
695        let _ = self
696            .raw()
697            .check_if_oom()
698            .map_err(|e| self.handle_hal_error(e));
699    }
700
701    pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
702        match error {
703            hal::DeviceError::OutOfMemory
704            | hal::DeviceError::Lost
705            | hal::DeviceError::Unexpected => {
706                self.lose(&error.to_string());
707            }
708        }
709        DeviceError::from_hal(error)
710    }
711
712    pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
713        match error {
714            hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
715            error => self.handle_hal_error(error),
716        }
717    }
718
719    /// Run some destroy operations that were deferred.
720    ///
721    /// Destroying the resources requires taking a write lock on the device's snatch lock,
722    /// so a good reason for deferring resource destruction is when we don't know for sure
723    /// how risky it is to take the lock (typically, it shouldn't be taken from the drop
724    /// implementation of a reference-counted structure).
725    /// The snatch lock must not be held while this function is called.
726    pub(crate) fn deferred_resource_destruction(&self) {
727        // Note that the deferred_destroy list may contain duplicate entries.
728        let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
729        for item in deferred_destroy {
730            match item {
731                DeferredDestroy::TextureViews(views) => {
732                    for view in views {
733                        let Some(view) = view.upgrade() else {
734                            continue;
735                        };
736                        let Some(raw_view) = view.raw.snatch(&mut self.snatchable_lock.write())
737                        else {
738                            continue;
739                        };
740
741                        resource_log!("Destroy raw {}", view.error_ident());
742
743                        unsafe {
744                            self.raw().destroy_texture_view(raw_view);
745                        }
746                    }
747                }
748                DeferredDestroy::BindGroups(bind_groups) => {
749                    for bind_group in bind_groups {
750                        let Some(bind_group) = bind_group.upgrade() else {
751                            continue;
752                        };
753                        let Some(raw_bind_group) =
754                            bind_group.raw.snatch(&mut self.snatchable_lock.write())
755                        else {
756                            continue;
757                        };
758
759                        resource_log!("Destroy raw {}", bind_group.error_ident());
760
761                        unsafe {
762                            self.raw().destroy_bind_group(raw_bind_group);
763                        }
764                    }
765                }
766            }
767        }
768    }
769
770    pub fn get_queue(&self) -> Option<Arc<Queue>> {
771        self.queue.get().as_ref()?.upgrade()
772    }
773
774    pub fn set_queue(&self, queue: &Arc<Queue>) {
775        assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
776    }
777
778    pub fn poll(
779        &self,
780        poll_type: wgt::PollType<crate::SubmissionIndex>,
781    ) -> Result<wgt::PollStatus, WaitIdleError> {
782        let (user_closures, result) = self.poll_and_return_closures(poll_type);
783        user_closures.fire();
784        result
785    }
786
787    /// Poll the device, returning any `UserClosures` that need to be executed.
788    ///
789    /// The caller must invoke the `UserClosures` even if this function returns
790    /// an error. This is an internal helper, used by `Device::poll` and
791    /// `Global::poll_all_devices`, so that `poll_all_devices` can invoke
792    /// closures once after all devices have been polled.
793    pub(crate) fn poll_and_return_closures(
794        &self,
795        poll_type: wgt::PollType<crate::SubmissionIndex>,
796    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
797        let snatch_guard = self.snatchable_lock.read();
798        let maintain_result = self.maintain(poll_type, snatch_guard);
799
800        self.lose_if_oom();
801
802        // Some deferred destroys are scheduled in maintain so run this right after
803        // to avoid holding on to them until the next device poll.
804        self.deferred_resource_destruction();
805
806        maintain_result
807    }
808
809    /// Check the current status of the GPU and process any submissions that have
810    /// finished.
811    ///
812    /// The `poll_type` argument tells if this function should wait for a particular
813    /// submission index to complete, or if it should just poll the current status.
814    ///
815    /// This will process _all_ completed submissions, even if the caller only asked
816    /// us to poll to a given submission index.
817    ///
818    /// Return a pair `(closures, result)`, where:
819    ///
820    /// - `closures` is a list of callbacks that need to be invoked informing the user
821    ///   about various things occurring. These happen and should be handled even if
822    ///   this function returns an error, hence they are outside of the result.
823    ///
824    /// - `results` is a boolean indicating the result of the wait operation, including
825    ///   if there was a timeout or a validation error.
826    pub(crate) fn maintain<'this>(
827        &'this self,
828        poll_type: wgt::PollType<crate::SubmissionIndex>,
829        snatch_guard: SnatchGuard,
830    ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
831        profiling::scope!("Device::maintain");
832
833        let mut user_closures = UserClosures::default();
834
835        // If a wait was requested, determine which submission index to wait for.
836        let wait_submission_index = match poll_type {
837            wgt::PollType::Wait {
838                submission_index: Some(submission_index),
839                ..
840            } => {
841                let last_successful_submission_index = self
842                    .last_successful_submission_index
843                    .load(Ordering::Acquire);
844
845                if submission_index > last_successful_submission_index {
846                    let result = Err(WaitIdleError::WrongSubmissionIndex(
847                        submission_index,
848                        last_successful_submission_index,
849                    ));
850
851                    return (user_closures, result);
852                }
853
854                Some(submission_index)
855            }
856            wgt::PollType::Wait {
857                submission_index: None,
858                ..
859            } => Some(
860                self.last_successful_submission_index
861                    .load(Ordering::Acquire),
862            ),
863            wgt::PollType::Poll => None,
864        };
865
866        // Wait for the submission index if requested.
867        if let Some(target_submission_index) = wait_submission_index {
868            log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
869
870            let wait_timeout = match poll_type {
871                wgt::PollType::Wait { timeout, .. } => timeout,
872                wgt::PollType::Poll => unreachable!(
873                    "`wait_submission_index` index for poll type `Poll` should be None"
874                ),
875            };
876
877            let wait_result = unsafe {
878                self.raw()
879                    .wait(self.fence.as_ref(), target_submission_index, wait_timeout)
880            };
881
882            // This error match is only about `DeviceErrors`. At this stage we do not care if
883            // the wait succeeded or not, and the `Ok(bool)`` variant is ignored.
884            if let Err(e) = wait_result {
885                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
886                return (user_closures, Err(hal_error));
887            }
888        }
889
890        // Get the currently finished submission index. This may be higher than the requested
891        // wait, or it may be less than the requested wait if the wait failed.
892        let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) };
893        let current_finished_submission = match fence_value_result {
894            Ok(fence_value) => fence_value,
895            Err(e) => {
896                let hal_error: WaitIdleError = self.handle_hal_error(e).into();
897                return (user_closures, Err(hal_error));
898            }
899        };
900
901        // Prevent new commands from being submitted as we want to act on `queue_empty`.
902        let command_indices = self.command_indices.read();
903        // Check that the device is valid. This is combined with queue empty to decide whether
904        // to destroy all resources. Queue.submit blocks on command indices being writable
905        // and rejects if invalid so if the device in now invalid, and all submissions are
906        // finished, there will be no more submissions.
907        let device_valid = self.is_valid();
908        drop(command_indices);
909
910        // Maintain all finished submissions on the queue, updating the relevant user closures and
911        // collecting if the queue is empty.
912        //
913        // We don't use the result of the wait here, as we want to progress forward as far as
914        // possible and the wait could have been for submissions that finished long ago.
915        let mut queue_empty = false;
916        if let Some(queue) = self.get_queue() {
917            let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
918            (
919                user_closures.submissions,
920                user_closures.mappings,
921                user_closures.blas_compact_ready,
922                queue_empty,
923            ) = queue_result;
924            // DEADLOCK PREVENTION: We must drop `snatch_guard` before `queue` goes out of scope.
925            //
926            // `Queue::drop` acquires the snatch guard. If we still hold it when `queue` is dropped
927            // at the end of this block, we would deadlock. This can happen in the following
928            // scenario:
929            //
930            // - Thread A calls `Device::maintain` while Thread B holds the last strong ref to the
931            //   queue.
932            // - Thread A calls `self.get_queue()`, obtaining a new strong ref, and enters this
933            //   branch.
934            // - Thread B drops its strong ref, making Thread A's ref the last one.
935            // - When `queue` goes out of scope here, `Queue::drop` runs and tries to acquire the
936            //   snatch guard — but Thread A (this thread) still holds it, causing a deadlock.
937            drop(snatch_guard);
938        } else {
939            drop(snatch_guard);
940        };
941
942        // Based on the queue empty status, and the current finished submission index, determine
943        // the result of the poll.
944        let result = if queue_empty {
945            if let Some(wait_submission_index) = wait_submission_index {
946                // Assert to ensure that if we received a queue empty status, the fence shows the
947                // correct value. This is defensive, as this should never be hit.
948                assert!(
949                    current_finished_submission >= wait_submission_index,
950                    concat!(
951                        "If the queue is empty, the current submission index ",
952                        "({}) should be at least the wait submission index ({})",
953                    ),
954                    current_finished_submission,
955                    wait_submission_index,
956                );
957            }
958
959            Ok(wgt::PollStatus::QueueEmpty)
960        } else if let Some(wait_submission_index) = wait_submission_index {
961            // This is theoretically possible to succeed more than checking on the poll result
962            // as submissions could have finished in the time between the timeout resolving,
963            // the thread getting scheduled again, and us checking the fence value.
964            if current_finished_submission >= wait_submission_index {
965                Ok(wgt::PollStatus::WaitSucceeded)
966            } else {
967                Err(WaitIdleError::Timeout)
968            }
969        } else {
970            Ok(wgt::PollStatus::Poll)
971        };
972
973        // Detect if we have been destroyed and now need to lose the device.
974        //
975        // If we are invalid (set at start of destroy) and our queue is empty,
976        // and we have a DeviceLostClosure, return the closure to be called by
977        // our caller. This will complete the steps for both destroy and for
978        // "lose the device".
979        let mut should_release_gpu_resource = false;
980        if !device_valid && queue_empty {
981            // We can release gpu resources associated with this device (but not
982            // while holding the life_tracker lock).
983            should_release_gpu_resource = true;
984
985            // If we have a DeviceLostClosure, build an invocation with the
986            // reason DeviceLostReason::Destroyed and no message.
987            if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
988                user_closures
989                    .device_lost_invocations
990                    .push(DeviceLostInvocation {
991                        closure: device_lost_closure,
992                        reason: DeviceLostReason::Destroyed,
993                        message: String::new(),
994                    });
995            }
996        }
997
998        if should_release_gpu_resource {
999            self.release_gpu_resources();
1000        }
1001
1002        (user_closures, result)
1003    }
1004
1005    pub fn create_buffer(
1006        self: &Arc<Self>,
1007        desc: &resource::BufferDescriptor,
1008    ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1009        self.check_is_valid()?;
1010
1011        if desc.size > self.limits.max_buffer_size {
1012            return Err(resource::CreateBufferError::MaxBufferSize {
1013                requested: desc.size,
1014                maximum: self.limits.max_buffer_size,
1015            });
1016        }
1017
1018        if desc
1019            .usage
1020            .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
1021        {
1022            self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)?;
1023        }
1024
1025        if desc.usage.contains(wgt::BufferUsages::INDEX)
1026            && desc.usage.contains(
1027                wgt::BufferUsages::VERTEX
1028                    | wgt::BufferUsages::UNIFORM
1029                    | wgt::BufferUsages::INDIRECT
1030                    | wgt::BufferUsages::STORAGE,
1031            )
1032        {
1033            self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
1034        }
1035
1036        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1037            return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
1038        }
1039
1040        if !self
1041            .features
1042            .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
1043        {
1044            use wgt::BufferUsages as Bu;
1045            let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
1046                && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
1047            let read_mismatch = desc.usage.contains(Bu::MAP_READ)
1048                && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
1049            if write_mismatch || read_mismatch {
1050                return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
1051            }
1052        }
1053
1054        let mut usage = conv::map_buffer_usage(desc.usage);
1055
1056        if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
1057            self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
1058            // We are going to be reading from it, internally;
1059            // when validating the content of the buffer
1060            usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
1061        }
1062
1063        if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
1064            usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
1065        }
1066
1067        if desc.mapped_at_creation {
1068            if !desc.size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1069                return Err(resource::CreateBufferError::UnalignedSize);
1070            }
1071            if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1072                // we are going to be copying into it, internally
1073                usage |= wgt::BufferUses::COPY_DST;
1074            }
1075        } else {
1076            // We are required to zero out (initialize) all memory. This is done
1077            // on demand using clear_buffer which requires write transfer usage!
1078            usage |= wgt::BufferUses::COPY_DST;
1079        }
1080
1081        let actual_size = if desc.size == 0 {
1082            wgt::COPY_BUFFER_ALIGNMENT
1083        } else if desc.usage.contains(wgt::BufferUsages::VERTEX) {
1084            // Bumping the size by 1 so that we can bind an empty range at the
1085            // end of the buffer.
1086            desc.size + 1
1087        } else {
1088            desc.size
1089        };
1090        let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT;
1091        let aligned_size = if clear_remainder != 0 {
1092            actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder
1093        } else {
1094            actual_size
1095        };
1096
1097        let hal_desc = hal::BufferDescriptor {
1098            label: desc.label.to_hal(self.instance_flags),
1099            size: aligned_size,
1100            usage,
1101            memory_flags: hal::MemoryFlags::empty(),
1102        };
1103        let buffer = unsafe { self.raw().create_buffer(&hal_desc) }
1104            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1105
1106        let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1107            // SAFETY: The size passed here must not overflow the buffer.
1108            self.timestamp_normalizer
1109                .get()
1110                .unwrap()
1111                .create_normalization_bind_group(
1112                    self,
1113                    &*buffer,
1114                    desc.label.as_deref(),
1115                    wgt::BufferSize::new(hal_desc.size).unwrap(),
1116                    desc.usage,
1117                )
1118        }?);
1119
1120        let indirect_validation_bind_groups =
1121            self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
1122
1123        let buffer = Buffer {
1124            raw: Snatchable::new(buffer),
1125            device: self.clone(),
1126            usage: desc.usage,
1127            size: desc.size,
1128            initialization_status: RwLock::new(
1129                rank::BUFFER_INITIALIZATION_STATUS,
1130                BufferInitTracker::new(aligned_size),
1131            ),
1132            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1133            destroyed_while_mapped: OnceCellOrLock::new(),
1134            label: desc.label.to_string(),
1135            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1136            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1137            timestamp_normalization_bind_group,
1138            indirect_validation_bind_groups,
1139        };
1140
1141        let buffer = Arc::new(buffer);
1142
1143        let buffer_use = if !desc.mapped_at_creation {
1144            wgt::BufferUses::empty()
1145        } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1146            // buffer is mappable, so we are just doing that at start
1147            let map_size = buffer.size;
1148            let mapping = if map_size == 0 {
1149                hal::BufferMapping {
1150                    ptr: core::ptr::NonNull::dangling(),
1151                    is_coherent: true,
1152                }
1153            } else {
1154                let snatch_guard: SnatchGuard = self.snatchable_lock.read();
1155                map_buffer(&buffer, 0, map_size, HostMap::Write, &snatch_guard)?
1156            };
1157            *buffer.map_state.lock() = resource::BufferMapState::Active {
1158                mapping,
1159                range: 0..map_size,
1160                host: HostMap::Write,
1161            };
1162            wgt::BufferUses::MAP_WRITE
1163        } else {
1164            let mut staging_buffer =
1165                StagingBuffer::new(self, wgt::BufferSize::new(aligned_size).unwrap())?;
1166
1167            // Zero initialize memory and then mark the buffer as initialized
1168            // (it's guaranteed that this is the case by the time the buffer is usable)
1169            staging_buffer.write_zeros();
1170            buffer.initialization_status.write().drain(0..aligned_size);
1171
1172            *buffer.map_state.lock() = resource::BufferMapState::Init { staging_buffer };
1173            wgt::BufferUses::COPY_DST
1174        };
1175
1176        self.trackers
1177            .lock()
1178            .buffers
1179            .insert_single(&buffer, buffer_use);
1180
1181        Ok(buffer)
1182    }
1183
1184    #[cfg(feature = "replay")]
1185    pub fn set_buffer_data(
1186        self: &Arc<Self>,
1187        buffer: &Arc<Buffer>,
1188        offset: wgt::BufferAddress,
1189        data: &[u8],
1190    ) -> resource::BufferAccessResult {
1191        use crate::resource::RawResourceAccess;
1192
1193        let device = &buffer.device;
1194
1195        device.check_is_valid()?;
1196        buffer.check_usage(wgt::BufferUsages::MAP_WRITE)?;
1197
1198        let last_submission = device
1199            .get_queue()
1200            .and_then(|queue| queue.lock_life().get_buffer_latest_submission_index(buffer));
1201
1202        if let Some(last_submission) = last_submission {
1203            device.wait_for_submit(last_submission)?;
1204        }
1205
1206        let snatch_guard = device.snatchable_lock.read();
1207        let raw_buf = buffer.try_raw(&snatch_guard)?;
1208
1209        if offset > buffer.size {
1210            return Err(resource::BufferAccessError::OutOfBoundsStartOffsetOverrun {
1211                index: offset,
1212                max: buffer.size,
1213            });
1214        } else if buffer.size - offset < u64::try_from(data.len()).unwrap() {
1215            return Err(resource::BufferAccessError::OutOfBoundsEndOffsetOverrun {
1216                index: offset,
1217                size: u64::try_from(data.len()).unwrap(),
1218                max: buffer.size,
1219            });
1220        }
1221
1222        let mapping = unsafe {
1223            device
1224                .raw()
1225                .map_buffer(raw_buf, offset..offset + u64::try_from(data.len()).unwrap())
1226        }
1227        .map_err(|e| device.handle_hal_error(e))?;
1228
1229        unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), mapping.ptr.as_ptr(), data.len()) };
1230
1231        if !mapping.is_coherent {
1232            #[allow(clippy::single_range_in_vec_init)]
1233            unsafe {
1234                device
1235                    .raw()
1236                    .flush_mapped_ranges(raw_buf, &[offset..offset + data.len() as u64])
1237            };
1238        }
1239
1240        unsafe { device.raw().unmap_buffer(raw_buf) };
1241
1242        Ok(())
1243    }
1244
1245    pub(crate) fn create_texture_from_hal(
1246        self: &Arc<Self>,
1247        hal_texture: Box<dyn hal::DynTexture>,
1248        desc: &resource::TextureDescriptor,
1249        initial_state: wgt::TextureUses,
1250    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1251        let format_features = self
1252            .describe_format_features(desc.format)
1253            .map_err(|error| resource::CreateTextureError::MissingFeatures(desc.format, error))?;
1254
1255        unsafe { self.raw().add_raw_texture(&*hal_texture) };
1256
1257        let texture = Texture::new(
1258            self,
1259            resource::TextureInner::Native { raw: hal_texture },
1260            conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1261            desc,
1262            format_features,
1263            resource::TextureClearMode::None,
1264            false,
1265        );
1266
1267        let texture = Arc::new(texture);
1268
1269        self.trackers
1270            .lock()
1271            .textures
1272            .insert_single(&texture, initial_state);
1273
1274        Ok(texture)
1275    }
1276
1277    /// # Safety
1278    ///
1279    /// - `hal_buffer` must have been created on this device.
1280    /// - `hal_buffer` must have been created respecting `desc` (in particular, the size).
1281    /// - `hal_buffer` must be initialized.
1282    /// - `hal_buffer` must not have zero size.
1283    pub(crate) unsafe fn create_buffer_from_hal(
1284        self: &Arc<Self>,
1285        hal_buffer: Box<dyn hal::DynBuffer>,
1286        desc: &resource::BufferDescriptor,
1287    ) -> (Fallible<Buffer>, Option<resource::CreateBufferError>) {
1288        let timestamp_normalization_bind_group = unsafe {
1289            match self
1290                .timestamp_normalizer
1291                .get()
1292                .unwrap()
1293                .create_normalization_bind_group(
1294                    self,
1295                    &*hal_buffer,
1296                    desc.label.as_deref(),
1297                    wgt::BufferSize::new(desc.size).unwrap(),
1298                    desc.usage,
1299                ) {
1300                Ok(bg) => Snatchable::new(bg),
1301                Err(e) => {
1302                    return (
1303                        Fallible::Invalid(Arc::new(desc.label.to_string())),
1304                        Some(e.into()),
1305                    )
1306                }
1307            }
1308        };
1309
1310        let indirect_validation_bind_groups = match self.create_indirect_validation_bind_groups(
1311            hal_buffer.as_ref(),
1312            desc.size,
1313            desc.usage,
1314        ) {
1315            Ok(ok) => ok,
1316            Err(e) => return (Fallible::Invalid(Arc::new(desc.label.to_string())), Some(e)),
1317        };
1318
1319        unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1320
1321        let buffer = Buffer {
1322            raw: Snatchable::new(hal_buffer),
1323            device: self.clone(),
1324            usage: desc.usage,
1325            size: desc.size,
1326            initialization_status: RwLock::new(
1327                rank::BUFFER_INITIALIZATION_STATUS,
1328                BufferInitTracker::new(0),
1329            ),
1330            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1331            destroyed_while_mapped: OnceCellOrLock::new(),
1332            label: desc.label.to_string(),
1333            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1334            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1335            timestamp_normalization_bind_group,
1336            indirect_validation_bind_groups,
1337        };
1338
1339        let buffer = Arc::new(buffer);
1340
1341        self.trackers
1342            .lock()
1343            .buffers
1344            .insert_single(&buffer, wgt::BufferUses::empty());
1345
1346        (Fallible::Valid(buffer), None)
1347    }
1348
1349    fn create_indirect_validation_bind_groups(
1350        &self,
1351        raw_buffer: &dyn hal::DynBuffer,
1352        buffer_size: u64,
1353        usage: wgt::BufferUsages,
1354    ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1355    {
1356        if !usage.contains(wgt::BufferUsages::INDIRECT) {
1357            return Ok(Snatchable::empty());
1358        }
1359
1360        let Some(ref indirect_validation) = self.indirect_validation else {
1361            return Ok(Snatchable::empty());
1362        };
1363
1364        let bind_groups = crate::indirect_validation::BindGroups::new(
1365            indirect_validation,
1366            self,
1367            buffer_size,
1368            raw_buffer,
1369        )
1370        .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1371
1372        if let Some(bind_groups) = bind_groups {
1373            Ok(Snatchable::new(bind_groups))
1374        } else {
1375            Ok(Snatchable::empty())
1376        }
1377    }
1378
1379    fn create_texture_inner(
1380        self: &Arc<Self>,
1381        desc: &resource::TextureDescriptor,
1382    ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1383        use resource::{CreateTextureError, TextureDimensionError};
1384
1385        self.check_is_valid()?;
1386
1387        if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1388            return Err(CreateTextureError::InvalidUsage(desc.usage));
1389        }
1390
1391        conv::check_texture_dimension_size(
1392            desc.dimension,
1393            desc.size,
1394            desc.sample_count,
1395            &self.limits,
1396        )?;
1397
1398        if desc.dimension != wgt::TextureDimension::D2 {
1399            // Depth textures can only be 2D
1400            if desc.format.is_depth_stencil_format() {
1401                return Err(CreateTextureError::InvalidDepthDimension(
1402                    desc.dimension,
1403                    desc.format,
1404                ));
1405            }
1406            // Transient textures can only be 2D
1407            if desc
1408                .usage
1409                .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1410            {
1411                return Err(CreateTextureError::InvalidDimensionUsages(
1412                    wgt::TextureUsages::TRANSIENT_ATTACHMENT,
1413                    desc.dimension,
1414                ));
1415            }
1416        }
1417
1418        if desc.dimension != wgt::TextureDimension::D2
1419            && desc.dimension != wgt::TextureDimension::D3
1420        {
1421            // Compressed textures can only be 2D or 3D
1422            if desc.format.is_compressed() {
1423                return Err(CreateTextureError::InvalidCompressedDimension(
1424                    desc.dimension,
1425                    desc.format,
1426                ));
1427            }
1428
1429            // Renderable textures can only be 2D or 3D
1430            if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1431                return Err(CreateTextureError::InvalidDimensionUsages(
1432                    wgt::TextureUsages::RENDER_ATTACHMENT,
1433                    desc.dimension,
1434                ));
1435            }
1436        }
1437
1438        if desc.format.is_compressed() {
1439            let (block_width, block_height) = desc.format.block_dimensions();
1440
1441            if !desc.size.width.is_multiple_of(block_width) {
1442                return Err(CreateTextureError::InvalidDimension(
1443                    TextureDimensionError::NotMultipleOfBlockWidth {
1444                        width: desc.size.width,
1445                        block_width,
1446                        format: desc.format,
1447                    },
1448                ));
1449            }
1450
1451            if !desc.size.height.is_multiple_of(block_height) {
1452                return Err(CreateTextureError::InvalidDimension(
1453                    TextureDimensionError::NotMultipleOfBlockHeight {
1454                        height: desc.size.height,
1455                        block_height,
1456                        format: desc.format,
1457                    },
1458                ));
1459            }
1460
1461            if desc.dimension == wgt::TextureDimension::D3 {
1462                // Only BCn formats with Sliced 3D feature can be used for 3D textures
1463                if desc.format.is_bcn() {
1464                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1465                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1466                } else if desc.format.is_astc() {
1467                    self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1468                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1469                } else {
1470                    return Err(CreateTextureError::InvalidCompressedDimension(
1471                        desc.dimension,
1472                        desc.format,
1473                    ));
1474                }
1475            }
1476        }
1477
1478        let mips = desc.mip_level_count;
1479        let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1480        if mips == 0 || mips > max_levels_allowed {
1481            return Err(CreateTextureError::InvalidMipLevelCount {
1482                requested: mips,
1483                maximum: max_levels_allowed,
1484            });
1485        }
1486
1487        {
1488            let (mut width_multiple, mut height_multiple) = desc.format.size_multiple_requirement();
1489
1490            if desc.format.is_multi_planar_format() {
1491                // TODO(https://github.com/gfx-rs/wgpu/issues/8491): fix
1492                // `mip_level_size` calculation for these formats and relax this
1493                // restriction.
1494                width_multiple <<= desc.mip_level_count.saturating_sub(1);
1495                height_multiple <<= desc.mip_level_count.saturating_sub(1);
1496            }
1497
1498            if !desc.size.width.is_multiple_of(width_multiple) {
1499                return Err(CreateTextureError::InvalidDimension(
1500                    TextureDimensionError::WidthNotMultipleOf {
1501                        width: desc.size.width,
1502                        multiple: width_multiple,
1503                        format: desc.format,
1504                    },
1505                ));
1506            }
1507
1508            if !desc.size.height.is_multiple_of(height_multiple) {
1509                return Err(CreateTextureError::InvalidDimension(
1510                    TextureDimensionError::HeightNotMultipleOf {
1511                        height: desc.size.height,
1512                        multiple: height_multiple,
1513                        format: desc.format,
1514                    },
1515                ));
1516            }
1517        }
1518
1519        if desc
1520            .usage
1521            .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1522        {
1523            if desc.usage
1524                != (wgt::TextureUsages::TRANSIENT_ATTACHMENT
1525                    | wgt::TextureUsages::RENDER_ATTACHMENT)
1526            {
1527                return Err(CreateTextureError::InvalidTransientTextureUsage(desc.usage));
1528            }
1529
1530            if desc.mip_level_count != 1 {
1531                return Err(CreateTextureError::InvalidTransientTextureMipLevelCount(
1532                    desc.mip_level_count,
1533                ));
1534            }
1535
1536            if desc.size.depth_or_array_layers != 1 {
1537                return Err(CreateTextureError::InvalidTransientTextureLayerCount(
1538                    desc.size.depth_or_array_layers,
1539                ));
1540            }
1541
1542            if !desc.view_formats.is_empty() {
1543                return Err(CreateTextureError::InvalidTransientTextureViewFormats);
1544            }
1545        }
1546
1547        let format_features = self
1548            .describe_format_features(desc.format)
1549            .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1550
1551        if desc.sample_count > 1 {
1552            // <https://www.w3.org/TR/2025/CRD-webgpu-20251120/#:~:text=If%20descriptor%2EsampleCount%20%3E%201>
1553            //
1554            // Note that there are also some checks related to the sample count
1555            // in [`conv::check_texture_dimension_size`].
1556
1557            if desc.mip_level_count != 1 {
1558                return Err(CreateTextureError::InvalidMipLevelCount {
1559                    requested: desc.mip_level_count,
1560                    maximum: 1,
1561                });
1562            }
1563
1564            if desc.size.depth_or_array_layers != 1
1565                && !self.features.contains(wgt::Features::MULTISAMPLE_ARRAY)
1566            {
1567                return Err(CreateTextureError::InvalidDimension(
1568                    TextureDimensionError::MultisampledDepthOrArrayLayer(
1569                        desc.size.depth_or_array_layers,
1570                    ),
1571                ));
1572            }
1573
1574            if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
1575                return Err(CreateTextureError::InvalidMultisampledStorageBinding);
1576            }
1577
1578            if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1579                return Err(CreateTextureError::MultisampledNotRenderAttachment);
1580            }
1581
1582            if !format_features.flags.intersects(
1583                wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
1584                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
1585                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
1586                    | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
1587            ) {
1588                return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
1589            }
1590
1591            if !format_features
1592                .flags
1593                .sample_count_supported(desc.sample_count)
1594            {
1595                return Err(CreateTextureError::InvalidSampleCount(
1596                    desc.sample_count,
1597                    desc.format,
1598                    desc.format
1599                        .guaranteed_format_features(self.features)
1600                        .flags
1601                        .supported_sample_counts(),
1602                    self.adapter
1603                        .get_texture_format_features(desc.format)
1604                        .flags
1605                        .supported_sample_counts(),
1606                ));
1607            };
1608        }
1609
1610        let missing_allowed_usages = match desc.format.planes() {
1611            Some(planes) => {
1612                let mut planes_usages = wgt::TextureUsages::all();
1613                for plane in 0..planes {
1614                    let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1615                    let format = desc.format.aspect_specific_format(aspect).unwrap();
1616                    let format_features = self
1617                        .describe_format_features(format)
1618                        .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1619
1620                    planes_usages &= format_features.allowed_usages;
1621                }
1622
1623                desc.usage - planes_usages
1624            }
1625            None => desc.usage - format_features.allowed_usages,
1626        };
1627
1628        if !missing_allowed_usages.is_empty() {
1629            // detect downlevel incompatibilities
1630            let wgpu_allowed_usages = desc
1631                .format
1632                .guaranteed_format_features(self.features)
1633                .allowed_usages;
1634            let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
1635            return Err(CreateTextureError::InvalidFormatUsages(
1636                missing_allowed_usages,
1637                desc.format,
1638                wgpu_missing_usages.is_empty(),
1639            ));
1640        }
1641
1642        let mut hal_view_formats = Vec::new();
1643        for format in desc.view_formats.iter() {
1644            if desc.format == *format {
1645                continue;
1646            }
1647            if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
1648                return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
1649            }
1650            hal_view_formats.push(*format);
1651        }
1652        if !hal_view_formats.is_empty() {
1653            self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
1654        }
1655
1656        let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
1657
1658        let hal_desc = hal::TextureDescriptor {
1659            label: desc.label.to_hal(self.instance_flags),
1660            size: desc.size,
1661            mip_level_count: desc.mip_level_count,
1662            sample_count: desc.sample_count,
1663            dimension: desc.dimension,
1664            format: desc.format,
1665            usage: hal_usage,
1666            memory_flags: hal::MemoryFlags::empty(),
1667            view_formats: hal_view_formats,
1668        };
1669
1670        let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
1671            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1672
1673        let clear_mode = if hal_usage
1674            .intersects(wgt::TextureUses::DEPTH_STENCIL_WRITE | wgt::TextureUses::COLOR_TARGET)
1675            && desc.dimension == wgt::TextureDimension::D2
1676        {
1677            let (is_color, usage) = if desc.format.is_depth_stencil_format() {
1678                (false, wgt::TextureUses::DEPTH_STENCIL_WRITE)
1679            } else {
1680                (true, wgt::TextureUses::COLOR_TARGET)
1681            };
1682
1683            let clear_label = hal_label(
1684                Some("(wgpu internal) clear texture view"),
1685                self.instance_flags,
1686            );
1687
1688            let mut clear_views = SmallVec::new();
1689            for mip_level in 0..desc.mip_level_count {
1690                for array_layer in 0..desc.size.depth_or_array_layers {
1691                    macro_rules! push_clear_view {
1692                        ($format:expr, $aspect:expr) => {
1693                            let desc = hal::TextureViewDescriptor {
1694                                label: clear_label,
1695                                format: $format,
1696                                dimension: TextureViewDimension::D2,
1697                                usage,
1698                                range: wgt::ImageSubresourceRange {
1699                                    aspect: $aspect,
1700                                    base_mip_level: mip_level,
1701                                    mip_level_count: Some(1),
1702                                    base_array_layer: array_layer,
1703                                    array_layer_count: Some(1),
1704                                },
1705                            };
1706                            clear_views.push(ManuallyDrop::new(
1707                                unsafe {
1708                                    self.raw().create_texture_view(raw_texture.as_ref(), &desc)
1709                                }
1710                                .map_err(|e| self.handle_hal_error(e))?,
1711                            ));
1712                        };
1713                    }
1714
1715                    if let Some(planes) = desc.format.planes() {
1716                        for plane in 0..planes {
1717                            let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1718                            let format = desc.format.aspect_specific_format(aspect).unwrap();
1719                            push_clear_view!(format, aspect);
1720                        }
1721                    } else {
1722                        push_clear_view!(desc.format, wgt::TextureAspect::All);
1723                    }
1724                }
1725            }
1726            resource::TextureClearMode::RenderPass {
1727                clear_views,
1728                is_color,
1729            }
1730        } else {
1731            resource::TextureClearMode::BufferCopy
1732        };
1733
1734        let texture = Texture::new(
1735            self,
1736            resource::TextureInner::Native { raw: raw_texture },
1737            hal_usage,
1738            desc,
1739            format_features,
1740            clear_mode,
1741            true,
1742        );
1743
1744        let texture = Arc::new(texture);
1745
1746        self.trackers
1747            .lock()
1748            .textures
1749            .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1750
1751        Ok(texture)
1752    }
1753
1754    pub fn create_texture(
1755        self: &Arc<Self>,
1756        desc: &resource::TextureDescriptor,
1757    ) -> (Arc<Texture>, Option<resource::CreateTextureError>) {
1758        let (texture, error) = match self.create_texture_inner(desc) {
1759            Ok(texture) => (texture, None),
1760            Err(e) => {
1761                let texture = Texture::invalid(self, desc);
1762                (Arc::new(texture), Some(e))
1763            }
1764        };
1765        api_log!(
1766            "Device::create_texture({desc:?}) -> {:?}",
1767            Arc::as_ptr(&texture)
1768        );
1769
1770        #[cfg(feature = "trace")]
1771        if let Some(ref mut trace) = *self.trace.lock() {
1772            use crate::device::trace::IntoTrace as _;
1773
1774            trace.add(trace::Action::CreateTexture(
1775                texture.to_trace(),
1776                desc.clone(),
1777            ));
1778        }
1779        (texture, error)
1780    }
1781
1782    /// Creates a texture that is guaranteed to be invalid
1783    pub fn create_texture_error(
1784        self: &Arc<Self>,
1785        desc: &resource::TextureDescriptor,
1786    ) -> Arc<Texture> {
1787        let texture = Arc::new(Texture::invalid(self, desc));
1788        #[cfg(feature = "trace")]
1789        if let Some(ref mut trace) = *self.trace.lock() {
1790            use crate::device::trace::IntoTrace as _;
1791
1792            trace.add(trace::Action::CreateTextureError(
1793                texture.to_trace(),
1794                desc.clone(),
1795            ));
1796        }
1797        texture
1798    }
1799
1800    pub fn create_texture_view(
1801        self: &Arc<Self>,
1802        texture: &Arc<Texture>,
1803        desc: &resource::TextureViewDescriptor,
1804    ) -> Result<Arc<TextureView>, resource::CreateTextureViewError> {
1805        self.check_is_valid()?;
1806
1807        let snatch_guard = texture.device.snatchable_lock.read();
1808
1809        let texture_raw = texture.try_inner(&snatch_guard)?.raw();
1810
1811        // resolve TextureViewDescriptor defaults
1812        // https://gpuweb.github.io/gpuweb/#abstract-opdef-resolving-gputextureviewdescriptor-defaults
1813        let resolved_format = desc.format.unwrap_or_else(|| {
1814            texture
1815                .desc
1816                .format
1817                .aspect_specific_format(desc.range.aspect)
1818                .unwrap_or(texture.desc.format)
1819        });
1820
1821        let resolved_dimension = desc
1822            .dimension
1823            .unwrap_or_else(|| match texture.desc.dimension {
1824                wgt::TextureDimension::D1 => TextureViewDimension::D1,
1825                wgt::TextureDimension::D2 => {
1826                    if texture.desc.array_layer_count() == 1 {
1827                        TextureViewDimension::D2
1828                    } else {
1829                        TextureViewDimension::D2Array
1830                    }
1831                }
1832                wgt::TextureDimension::D3 => TextureViewDimension::D3,
1833            });
1834
1835        let resolved_mip_level_count = desc.range.mip_level_count.unwrap_or_else(|| {
1836            texture
1837                .desc
1838                .mip_level_count
1839                .saturating_sub(desc.range.base_mip_level)
1840        });
1841
1842        let resolved_array_layer_count =
1843            desc.range
1844                .array_layer_count
1845                .unwrap_or_else(|| match resolved_dimension {
1846                    TextureViewDimension::D1
1847                    | TextureViewDimension::D2
1848                    | TextureViewDimension::D3 => 1,
1849                    TextureViewDimension::Cube => 6,
1850                    TextureViewDimension::D2Array | TextureViewDimension::CubeArray => texture
1851                        .desc
1852                        .array_layer_count()
1853                        .saturating_sub(desc.range.base_array_layer),
1854                });
1855
1856        let resolved_usage = {
1857            let usage = desc.usage.unwrap_or(wgt::TextureUsages::empty());
1858            if usage.is_empty() {
1859                texture.desc.usage
1860            } else if texture.desc.usage.contains(usage) {
1861                // Transient texture usage subsetting is disallowed
1862                if texture
1863                    .desc
1864                    .usage
1865                    .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1866                    && texture.desc.usage != usage
1867                {
1868                    return Err(
1869                        resource::CreateTextureViewError::InvalidTransientTextureViewUsage {
1870                            texture: texture.desc.usage,
1871                            view: usage,
1872                        },
1873                    );
1874                }
1875
1876                usage
1877            } else {
1878                return Err(resource::CreateTextureViewError::InvalidTextureViewUsage {
1879                    view: usage,
1880                    texture: texture.desc.usage,
1881                });
1882            }
1883        };
1884
1885        let format_features = self.describe_format_features(resolved_format)?;
1886        let allowed_format_usages = format_features.allowed_usages;
1887        if resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1888            && !allowed_format_usages.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1889        {
1890            return Err(
1891                resource::CreateTextureViewError::TextureViewFormatNotRenderable(resolved_format),
1892            );
1893        }
1894
1895        if resolved_usage.contains(wgt::TextureUsages::STORAGE_BINDING)
1896            && !allowed_format_usages.contains(wgt::TextureUsages::STORAGE_BINDING)
1897        {
1898            return Err(
1899                resource::CreateTextureViewError::TextureViewFormatNotStorage(resolved_format),
1900            );
1901        }
1902
1903        // validate TextureViewDescriptor
1904
1905        let aspects = hal::FormatAspects::new(texture.desc.format, desc.range.aspect);
1906        if aspects.is_empty() {
1907            return Err(resource::CreateTextureViewError::InvalidAspect {
1908                texture_format: texture.desc.format,
1909                requested_aspect: desc.range.aspect,
1910            });
1911        }
1912
1913        let format_is_good = if desc.range.aspect == wgt::TextureAspect::All {
1914            resolved_format == texture.desc.format
1915                || texture.desc.view_formats.contains(&resolved_format)
1916        } else {
1917            Some(resolved_format)
1918                == texture
1919                    .desc
1920                    .format
1921                    .aspect_specific_format(desc.range.aspect)
1922        };
1923        if !format_is_good {
1924            return Err(resource::CreateTextureViewError::FormatReinterpretation {
1925                texture: texture.desc.format,
1926                view: resolved_format,
1927            });
1928        }
1929
1930        // check if multisampled texture is seen as anything but 2D
1931        if texture.desc.sample_count > 1 && resolved_dimension != TextureViewDimension::D2 {
1932            // Multisample is allowed on 2D arrays, only if explicitly supported
1933            let multisample_array_exception = resolved_dimension == TextureViewDimension::D2Array
1934                && self.features.contains(wgt::Features::MULTISAMPLE_ARRAY);
1935
1936            if !multisample_array_exception {
1937                return Err(
1938                    resource::CreateTextureViewError::InvalidMultisampledTextureViewDimension(
1939                        resolved_dimension,
1940                    ),
1941                );
1942            }
1943        }
1944
1945        // check if the dimension is compatible with the texture
1946        if texture.desc.dimension != resolved_dimension.compatible_texture_dimension() {
1947            return Err(
1948                resource::CreateTextureViewError::InvalidTextureViewDimension {
1949                    view: resolved_dimension,
1950                    texture: texture.desc.dimension,
1951                },
1952            );
1953        }
1954
1955        match resolved_dimension {
1956            TextureViewDimension::D1 | TextureViewDimension::D2 | TextureViewDimension::D3 => {
1957                if resolved_array_layer_count != 1 {
1958                    return Err(resource::CreateTextureViewError::InvalidArrayLayerCount {
1959                        requested: resolved_array_layer_count,
1960                        dim: resolved_dimension,
1961                    });
1962                }
1963            }
1964            TextureViewDimension::Cube => {
1965                if resolved_array_layer_count != 6 {
1966                    return Err(
1967                        resource::CreateTextureViewError::InvalidCubemapTextureDepth {
1968                            depth: resolved_array_layer_count,
1969                        },
1970                    );
1971                }
1972            }
1973            TextureViewDimension::CubeArray => {
1974                if !resolved_array_layer_count.is_multiple_of(6) {
1975                    return Err(
1976                        resource::CreateTextureViewError::InvalidCubemapArrayTextureDepth {
1977                            depth: resolved_array_layer_count,
1978                        },
1979                    );
1980                }
1981            }
1982            _ => {}
1983        }
1984
1985        match resolved_dimension {
1986            TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1987                if texture.desc.size.width != texture.desc.size.height {
1988                    return Err(resource::CreateTextureViewError::InvalidCubeTextureViewSize);
1989                }
1990            }
1991            _ => {}
1992        }
1993
1994        if resolved_mip_level_count == 0 {
1995            return Err(resource::CreateTextureViewError::ZeroMipLevelCount);
1996        }
1997
1998        let mip_level_end = desc
1999            .range
2000            .base_mip_level
2001            .saturating_add(resolved_mip_level_count);
2002
2003        let level_end = texture.desc.mip_level_count;
2004        if mip_level_end > level_end {
2005            return Err(resource::CreateTextureViewError::TooManyMipLevels {
2006                base_mip_level: desc.range.base_mip_level,
2007                mip_level_count: resolved_mip_level_count,
2008                total: level_end,
2009            });
2010        }
2011
2012        if resolved_array_layer_count == 0 {
2013            return Err(resource::CreateTextureViewError::ZeroArrayLayerCount);
2014        }
2015
2016        let array_layer_end = desc
2017            .range
2018            .base_array_layer
2019            .saturating_add(resolved_array_layer_count);
2020
2021        let layer_end = texture.desc.array_layer_count();
2022        if array_layer_end > layer_end {
2023            return Err(resource::CreateTextureViewError::TooManyArrayLayers {
2024                base_array_layer: desc.range.base_array_layer,
2025                array_layer_count: resolved_array_layer_count,
2026                total: layer_end,
2027            });
2028        };
2029
2030        // https://gpuweb.github.io/gpuweb/#abstract-opdef-renderable-texture-view
2031        let render_extent = 'error: {
2032            if !resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
2033                break 'error Err(TextureViewNotRenderableReason::Usage(resolved_usage));
2034            }
2035
2036            let allowed_view_dimensions = [
2037                TextureViewDimension::D2,
2038                TextureViewDimension::D2Array,
2039                TextureViewDimension::D3,
2040            ];
2041            if !allowed_view_dimensions.contains(&resolved_dimension) {
2042                break 'error Err(TextureViewNotRenderableReason::Dimension(
2043                    resolved_dimension,
2044                ));
2045            }
2046
2047            if resolved_mip_level_count != 1 {
2048                break 'error Err(TextureViewNotRenderableReason::MipLevelCount(
2049                    resolved_mip_level_count,
2050                ));
2051            }
2052
2053            if resolved_array_layer_count != 1
2054                && !(self.features.contains(wgt::Features::MULTIVIEW))
2055            {
2056                break 'error Err(TextureViewNotRenderableReason::ArrayLayerCount(
2057                    resolved_array_layer_count,
2058                ));
2059            }
2060
2061            if !texture.desc.format.is_multi_planar_format()
2062                && aspects != hal::FormatAspects::from(texture.desc.format)
2063            {
2064                break 'error Err(TextureViewNotRenderableReason::Aspects(aspects));
2065            }
2066
2067            Ok(texture
2068                .desc
2069                .compute_render_extent(desc.range.base_mip_level, desc.range.aspect.to_plane()))
2070        };
2071
2072        // filter the usages based on the other criteria
2073        let usage = {
2074            let resolved_hal_usage = conv::map_texture_usage(
2075                resolved_usage,
2076                resolved_format.into(),
2077                format_features.flags,
2078            );
2079            let mask_copy = !(wgt::TextureUses::COPY_SRC | wgt::TextureUses::COPY_DST);
2080            let mask_dimension = match resolved_dimension {
2081                TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2082                    wgt::TextureUses::RESOURCE
2083                }
2084                TextureViewDimension::D3 => {
2085                    wgt::TextureUses::RESOURCE
2086                        | wgt::TextureUses::STORAGE_READ_ONLY
2087                        | wgt::TextureUses::STORAGE_WRITE_ONLY
2088                        | wgt::TextureUses::STORAGE_READ_WRITE
2089                }
2090                _ => wgt::TextureUses::all(),
2091            };
2092            let mask_mip_level = if resolved_mip_level_count == 1 {
2093                wgt::TextureUses::all()
2094            } else {
2095                wgt::TextureUses::RESOURCE
2096            };
2097            resolved_hal_usage & mask_copy & mask_dimension & mask_mip_level
2098        };
2099
2100        // use the combined depth-stencil format for the view
2101        let format = if resolved_format.is_depth_stencil_component(texture.desc.format) {
2102            texture.desc.format
2103        } else {
2104            resolved_format
2105        };
2106
2107        let resolved_range = wgt::ImageSubresourceRange {
2108            aspect: desc.range.aspect,
2109            base_mip_level: desc.range.base_mip_level,
2110            mip_level_count: Some(resolved_mip_level_count),
2111            base_array_layer: desc.range.base_array_layer,
2112            array_layer_count: Some(resolved_array_layer_count),
2113        };
2114
2115        let hal_desc = hal::TextureViewDescriptor {
2116            label: desc.label.to_hal(self.instance_flags),
2117            format,
2118            dimension: resolved_dimension,
2119            usage,
2120            range: resolved_range,
2121        };
2122
2123        let raw = unsafe { self.raw().create_texture_view(texture_raw, &hal_desc) }
2124            .map_err(|e| self.handle_hal_error(e))?;
2125
2126        let selector = TextureSelector {
2127            mips: desc.range.base_mip_level..mip_level_end,
2128            layers: desc.range.base_array_layer..array_layer_end,
2129        };
2130
2131        let view = TextureView {
2132            raw: Snatchable::new(raw),
2133            parent: texture.clone(),
2134            device: self.clone(),
2135            desc: resource::HalTextureViewDescriptor {
2136                texture_format: texture.desc.format,
2137                format: resolved_format,
2138                dimension: resolved_dimension,
2139                usage: resolved_usage,
2140                range: resolved_range,
2141            },
2142            format_features: texture.format_features,
2143            render_extent,
2144            samples: texture.desc.sample_count,
2145            selector,
2146            label: desc.label.to_string(),
2147        };
2148
2149        let view = Arc::new(view);
2150
2151        {
2152            let mut views = texture.views.lock();
2153            views.push(Arc::downgrade(&view));
2154        }
2155
2156        Ok(view)
2157    }
2158
2159    pub fn create_external_texture(
2160        self: &Arc<Self>,
2161        desc: &resource::ExternalTextureDescriptor,
2162        planes: &[Arc<TextureView>],
2163    ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
2164        use resource::CreateExternalTextureError;
2165        self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
2166        self.check_is_valid()?;
2167
2168        if desc.num_planes() != planes.len() {
2169            return Err(CreateExternalTextureError::IncorrectPlaneCount {
2170                format: desc.format,
2171                expected: desc.num_planes(),
2172                provided: planes.len(),
2173            });
2174        }
2175
2176        let planes = planes
2177            .iter()
2178            .enumerate()
2179            .map(|(i, plane)| {
2180                if plane.samples != 1 {
2181                    return Err(CreateExternalTextureError::InvalidPlaneMultisample(
2182                        plane.samples,
2183                    ));
2184                }
2185
2186                let sample_type = plane
2187                    .desc
2188                    .format
2189                    .sample_type(Some(plane.desc.range.aspect), Some(self.features))
2190                    .unwrap();
2191                if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
2192                    return Err(CreateExternalTextureError::InvalidPlaneSampleType {
2193                        format: plane.desc.format,
2194                        sample_type,
2195                    });
2196                }
2197
2198                if plane.desc.dimension != TextureViewDimension::D2 {
2199                    return Err(CreateExternalTextureError::InvalidPlaneDimension(
2200                        plane.desc.dimension,
2201                    ));
2202                }
2203
2204                let expected_components = match desc.format {
2205                    wgt::ExternalTextureFormat::Rgba => 4,
2206                    wgt::ExternalTextureFormat::Nv12 => match i {
2207                        0 => 1,
2208                        1 => 2,
2209                        _ => unreachable!(),
2210                    },
2211                    wgt::ExternalTextureFormat::Yu12 => 1,
2212                };
2213                if plane.desc.format.components() != expected_components {
2214                    return Err(CreateExternalTextureError::InvalidPlaneFormat {
2215                        format: desc.format,
2216                        plane: i,
2217                        expected: expected_components,
2218                        provided: plane.desc.format,
2219                    });
2220                }
2221
2222                plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
2223                Ok(plane.clone())
2224            })
2225            .collect::<Result<_, _>>()?;
2226
2227        let params_data = ExternalTextureParams::from_desc(desc);
2228        let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
2229        let params_desc = resource::BufferDescriptor {
2230            label: label.map(Cow::Owned),
2231            size: size_of_val(&params_data) as wgt::BufferAddress,
2232            usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
2233            mapped_at_creation: false,
2234        };
2235        let params = self.create_buffer(&params_desc)?;
2236        self.get_queue().unwrap().write_buffer(
2237            params.clone(),
2238            0,
2239            bytemuck::bytes_of(&params_data),
2240        )?;
2241
2242        let external_texture = ExternalTexture {
2243            device: self.clone(),
2244            planes,
2245            params,
2246            label: desc.label.to_string(),
2247            tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
2248        };
2249        let external_texture = Arc::new(external_texture);
2250
2251        Ok(external_texture)
2252    }
2253
2254    pub fn create_sampler(
2255        self: &Arc<Self>,
2256        desc: &resource::SamplerDescriptor,
2257    ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
2258        self.check_is_valid()?;
2259
2260        if desc
2261            .address_modes
2262            .iter()
2263            .any(|am| am == &wgt::AddressMode::ClampToBorder)
2264        {
2265            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
2266        }
2267
2268        if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
2269            self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
2270        }
2271
2272        if desc.lod_min_clamp < 0.0 {
2273            return Err(resource::CreateSamplerError::InvalidLodMinClamp(
2274                desc.lod_min_clamp,
2275            ));
2276        }
2277        if desc.lod_max_clamp < desc.lod_min_clamp {
2278            return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
2279                lod_min_clamp: desc.lod_min_clamp,
2280                lod_max_clamp: desc.lod_max_clamp,
2281            });
2282        }
2283
2284        if desc.anisotropy_clamp < 1 {
2285            return Err(resource::CreateSamplerError::InvalidAnisotropy(
2286                desc.anisotropy_clamp,
2287            ));
2288        }
2289
2290        if desc.anisotropy_clamp != 1 {
2291            if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
2292                return Err(
2293                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2294                        filter_type: resource::SamplerFilterErrorType::MinFilter,
2295                        filter_mode: desc.min_filter,
2296                        anisotropic_clamp: desc.anisotropy_clamp,
2297                    },
2298                );
2299            }
2300            if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
2301                return Err(
2302                    resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2303                        filter_type: resource::SamplerFilterErrorType::MagFilter,
2304                        filter_mode: desc.mag_filter,
2305                        anisotropic_clamp: desc.anisotropy_clamp,
2306                    },
2307                );
2308            }
2309            if !matches!(desc.mipmap_filter, wgt::MipmapFilterMode::Linear) {
2310                return Err(
2311                    resource::CreateSamplerError::InvalidMipmapFilterModeWithAnisotropy {
2312                        filter_type: resource::SamplerFilterErrorType::MipmapFilter,
2313                        filter_mode: desc.mipmap_filter,
2314                        anisotropic_clamp: desc.anisotropy_clamp,
2315                    },
2316                );
2317            }
2318        }
2319
2320        let anisotropy_clamp = if self
2321            .downlevel
2322            .flags
2323            .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
2324        {
2325            // Clamp anisotropy clamp to [1, 16] per the wgpu-hal interface
2326            desc.anisotropy_clamp.min(16)
2327        } else {
2328            // If it isn't supported, set this unconditionally to 1
2329            1
2330        };
2331
2332        //TODO: check for wgt::DownlevelFlags::COMPARISON_SAMPLERS
2333
2334        let hal_desc = hal::SamplerDescriptor {
2335            label: desc.label.to_hal(self.instance_flags),
2336            address_modes: desc.address_modes,
2337            mag_filter: desc.mag_filter,
2338            min_filter: desc.min_filter,
2339            mipmap_filter: desc.mipmap_filter,
2340            lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
2341            compare: desc.compare,
2342            anisotropy_clamp,
2343            border_color: desc.border_color,
2344        };
2345
2346        let raw = unsafe { self.raw().create_sampler(&hal_desc) }
2347            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2348
2349        let sampler = Sampler {
2350            raw: ManuallyDrop::new(raw),
2351            device: self.clone(),
2352            label: desc.label.to_string(),
2353            tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
2354            comparison: desc.compare.is_some(),
2355            filtering: desc.min_filter == wgt::FilterMode::Linear
2356                || desc.mag_filter == wgt::FilterMode::Linear
2357                || desc.mipmap_filter == wgt::MipmapFilterMode::Linear,
2358        };
2359
2360        let sampler = Arc::new(sampler);
2361
2362        Ok(sampler)
2363    }
2364
2365    pub fn create_shader_module<'a>(
2366        self: &Arc<Self>,
2367        desc: &pipeline::ShaderModuleDescriptor<'a>,
2368        source: pipeline::ShaderModuleSource<'a>,
2369    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2370        self.check_is_valid()?;
2371
2372        let (module, source) = match source {
2373            #[cfg(feature = "wgsl")]
2374            pipeline::ShaderModuleSource::Wgsl(code) => {
2375                profiling::scope!("naga::front::wgsl::parse");
2376                let capabilities =
2377                    features_to_naga_capabilities(self.features, self.downlevel.flags);
2378                let mut options = naga::front::wgsl::Options::new();
2379                options.capabilities = capabilities;
2380                let mut frontend = naga::front::wgsl::Frontend::new_with_options(options);
2381                let module = frontend.parse(&code).map_err(|inner| {
2382                    pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2383                        source: code.to_string(),
2384                        label: desc.label.as_ref().map(|l| l.to_string()),
2385                        inner: Box::new(inner),
2386                    })
2387                })?;
2388                (Cow::Owned(module), code.into_owned())
2389            }
2390            #[cfg(feature = "spirv")]
2391            pipeline::ShaderModuleSource::SpirV(spv, options) => {
2392                let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2393                profiling::scope!("naga::front::spv::Frontend");
2394                let module = parser.parse().map_err(|inner| {
2395                    pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2396                        source: String::new(),
2397                        label: desc.label.as_ref().map(|l| l.to_string()),
2398                        inner: Box::new(inner),
2399                    })
2400                })?;
2401                (Cow::Owned(module), String::new())
2402            }
2403            #[cfg(feature = "glsl")]
2404            pipeline::ShaderModuleSource::Glsl(code, options) => {
2405                let mut parser = naga::front::glsl::Frontend::default();
2406                profiling::scope!("naga::front::glsl::Frontend.parse");
2407                let module = parser.parse(&options, &code).map_err(|inner| {
2408                    pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2409                        source: code.to_string(),
2410                        label: desc.label.as_ref().map(|l| l.to_string()),
2411                        inner: Box::new(inner),
2412                    })
2413                })?;
2414                (Cow::Owned(module), code.into_owned())
2415            }
2416            pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2417            pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2418        };
2419        for (_, var) in module.global_variables.iter() {
2420            match var.binding {
2421                Some(br) if br.group >= self.limits.max_bind_groups => {
2422                    return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2423                        bind: br,
2424                        group: br.group,
2425                        limit: self.limits.max_bind_groups,
2426                    });
2427                }
2428                _ => continue,
2429            };
2430        }
2431
2432        profiling::scope!("naga::validate");
2433        let debug_source =
2434            if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2435                Some(hal::DebugSource {
2436                    file_name: Cow::Owned(
2437                        desc.label
2438                            .as_ref()
2439                            .map_or("shader".to_string(), |l| l.to_string()),
2440                    ),
2441                    source_code: Cow::Owned(source.clone()),
2442                })
2443            } else {
2444                None
2445            };
2446
2447        let info = create_validator(
2448            self.features,
2449            self.downlevel.flags,
2450            naga::valid::ValidationFlags::all(),
2451        )
2452        .validate(&module)
2453        .map_err(|inner| {
2454            pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2455                source,
2456                label: desc.label.as_ref().map(|l| l.to_string()),
2457                inner: Box::new(inner),
2458            })
2459        })?;
2460
2461        let interface = validation::Interface::new(&module, &info, self.limits.clone());
2462        let hal_shader = hal::ShaderInput::Naga(hal::NagaShader {
2463            module,
2464            info,
2465            debug_source,
2466        });
2467        let hal_desc = hal::ShaderModuleDescriptor {
2468            label: desc.label.to_hal(self.instance_flags),
2469            runtime_checks: desc.runtime_checks,
2470        };
2471        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2472            Ok(raw) => raw,
2473            Err(error) => {
2474                return Err(match error {
2475                    hal::ShaderError::Device(error) => {
2476                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2477                    }
2478                    hal::ShaderError::Compilation(ref msg) => {
2479                        log::error!("Shader error: {msg}");
2480                        pipeline::CreateShaderModuleError::Generation
2481                    }
2482                })
2483            }
2484        };
2485
2486        let module = pipeline::ShaderModule {
2487            raw: ManuallyDrop::new(raw),
2488            device: self.clone(),
2489            interface: ShaderMetaData::Interface(interface),
2490            label: desc.label.to_string(),
2491        };
2492
2493        let module = Arc::new(module);
2494
2495        Ok(module)
2496    }
2497
2498    /// Not a public API. For use by `player` only.
2499    #[allow(unused_unsafe)]
2500    #[doc(hidden)]
2501    pub unsafe fn create_shader_module_passthrough<'a>(
2502        self: &Arc<Self>,
2503        descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2504    ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2505        self.check_is_valid()?;
2506        self.require_features(wgt::Features::PASSTHROUGH_SHADERS)?;
2507
2508        // Mainly important for GLSL or SPIR-V or DXIL, which each take exactly 1 entry point.
2509        if (descriptor.dxil.is_some() || descriptor.glsl.is_some())
2510            && descriptor.entry_points.len() != 1
2511        {
2512            return Err(pipeline::CreateShaderModuleError::IncorrectPassthroughEntryPointCount);
2513        }
2514
2515        let entry_point_hashmap = || {
2516            descriptor
2517                .entry_points
2518                .iter()
2519                .map(|e| (e.name.to_string(), e.workgroup_size))
2520                .collect()
2521        };
2522
2523        let hal_shader = match self.backend() {
2524            wgt::Backend::Vulkan => hal::ShaderInput::SpirV(
2525                descriptor
2526                    .spirv
2527                    .as_ref()
2528                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2529            ),
2530            wgt::Backend::Dx12 => {
2531                if let Some(dxil) = &descriptor.dxil {
2532                    hal::ShaderInput::Dxil { shader: dxil }
2533                } else if let Some(hlsl) = &descriptor.hlsl {
2534                    hal::ShaderInput::Hlsl { shader: hlsl }
2535                } else {
2536                    return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2537                }
2538            }
2539            wgt::Backend::Metal => {
2540                if let Some(metallib) = &descriptor.metallib {
2541                    hal::ShaderInput::MetalLib {
2542                        file: metallib,
2543                        num_workgroups: entry_point_hashmap(),
2544                    }
2545                } else if let Some(msl) = &descriptor.msl {
2546                    hal::ShaderInput::Msl {
2547                        shader: msl,
2548                        num_workgroups: entry_point_hashmap(),
2549                    }
2550                } else {
2551                    return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2552                }
2553            }
2554            wgt::Backend::Gl => hal::ShaderInput::Glsl {
2555                shader: descriptor
2556                    .glsl
2557                    .as_ref()
2558                    .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2559            },
2560            wgt::Backend::Noop => {
2561                return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend)
2562            }
2563            wgt::Backend::BrowserWebGpu => unreachable!(),
2564        };
2565
2566        let hal_desc = hal::ShaderModuleDescriptor {
2567            label: descriptor.label.to_hal(self.instance_flags),
2568            runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2569        };
2570
2571        let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2572            Ok(raw) => raw,
2573            Err(error) => {
2574                return Err(match error {
2575                    hal::ShaderError::Device(error) => {
2576                        pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2577                    }
2578                    hal::ShaderError::Compilation(ref msg) => {
2579                        log::error!("Shader error: {msg}");
2580                        pipeline::CreateShaderModuleError::Generation
2581                    }
2582                })
2583            }
2584        };
2585
2586        let module = pipeline::ShaderModule {
2587            raw: ManuallyDrop::new(raw),
2588            device: self.clone(),
2589            interface: ShaderMetaData::Passthrough(PassthroughInterface {
2590                entry_point_names: descriptor
2591                    .entry_points
2592                    .iter()
2593                    .map(|e| e.name.to_string())
2594                    .collect(),
2595            }),
2596            label: descriptor.label.to_string(),
2597        };
2598
2599        Ok(Arc::new(module))
2600    }
2601
2602    pub(crate) fn create_command_encoder(
2603        self: &Arc<Self>,
2604        label: &crate::Label,
2605    ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2606        self.check_is_valid()?;
2607
2608        let queue = self.get_queue().unwrap();
2609
2610        let encoder = self
2611            .command_allocator
2612            .acquire_encoder(self.raw(), queue.raw())
2613            .map_err(|e| self.handle_hal_error(e))?;
2614
2615        let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2616
2617        let cmd_enc = Arc::new(cmd_enc);
2618
2619        Ok(cmd_enc)
2620    }
2621
2622    /// Generate information about late-validated buffer bindings for pipelines.
2623    //TODO: should this be combined with `get_introspection_bind_group_layouts` in some way?
2624    fn make_late_sized_buffer_groups(
2625        shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2626        layout: &binding_model::PipelineLayout,
2627    ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2628        // Given the shader-required binding sizes and the pipeline layout,
2629        // return the filtered list of them in the layout order,
2630        // removing those with given `min_binding_size`.
2631        layout
2632            .bind_group_layouts
2633            .iter()
2634            .enumerate()
2635            .map(|(group_index, bgl)| {
2636                let Some(bgl) = bgl else {
2637                    return pipeline::LateSizedBufferGroup::default();
2638                };
2639
2640                let shader_sizes = bgl
2641                    .entries
2642                    .values()
2643                    .filter_map(|entry| match entry.ty {
2644                        wgt::BindingType::Buffer {
2645                            min_binding_size: None,
2646                            ..
2647                        } => {
2648                            let rb = naga::ResourceBinding {
2649                                group: group_index as u32,
2650                                binding: entry.binding,
2651                            };
2652                            let shader_size =
2653                                shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
2654                            Some(shader_size)
2655                        }
2656                        _ => None,
2657                    })
2658                    .collect();
2659                pipeline::LateSizedBufferGroup { shader_sizes }
2660            })
2661            .collect()
2662    }
2663
2664    pub fn create_bind_group_layout(
2665        self: &Arc<Self>,
2666        desc: &binding_model::BindGroupLayoutDescriptor,
2667    ) -> (Arc<BindGroupLayout>, Option<CreateBindGroupLayoutError>) {
2668        let (bgl, error) = match self.create_bind_group_layout_inner(desc) {
2669            Ok(layout) => (layout, None),
2670            Err(e) => (
2671                BindGroupLayout::invalid(self, desc.label.to_string()),
2672                Some(e),
2673            ),
2674        };
2675        #[cfg(feature = "trace")]
2676        if let Some(ref mut trace) = *self.trace.lock() {
2677            use crate::device::trace::IntoTrace;
2678
2679            trace.add(trace::Action::CreateBindGroupLayout(
2680                bgl.to_trace(),
2681                desc.clone(),
2682            ));
2683        }
2684        (bgl, error)
2685    }
2686
2687    fn create_bind_group_layout_inner(
2688        self: &Arc<Device>,
2689        desc: &binding_model::BindGroupLayoutDescriptor,
2690    ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2691        self.check_is_valid()?;
2692
2693        let entry_map = bgl::EntryMap::from_entries(&desc.entries)?;
2694
2695        let bgl_result = self.bgl_pool.get_or_init(entry_map, |entry_map| {
2696            let bgl =
2697                self.create_bind_group_layout_impl(&desc.label, entry_map, bgl::Origin::Pool)?;
2698            bgl.exclusive_pipeline
2699                .set(binding_model::ExclusivePipeline::None)
2700                .unwrap();
2701            Ok(bgl)
2702        });
2703
2704        match bgl_result {
2705            Ok(layout) => Ok(layout),
2706            Err(e) => Err(e),
2707        }
2708    }
2709
2710    fn create_bind_group_layout_impl(
2711        self: &Arc<Self>,
2712        label: &crate::Label,
2713        entry_map: bgl::EntryMap,
2714        origin: bgl::Origin,
2715    ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2716        #[derive(PartialEq)]
2717        enum WritableStorage {
2718            Yes,
2719            No,
2720        }
2721
2722        for entry in entry_map.values() {
2723            if entry.binding >= self.limits.max_bindings_per_bind_group {
2724                return Err(CreateBindGroupLayoutError::InvalidBindingIndex {
2725                    binding: entry.binding,
2726                    maximum: self.limits.max_bindings_per_bind_group,
2727                });
2728            }
2729
2730            use wgt::BindingType as Bt;
2731
2732            let mut required_features = wgt::Features::empty();
2733            let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
2734            let (array_feature, writable_storage) = match entry.ty {
2735                Bt::Buffer {
2736                    ty: wgt::BufferBindingType::Uniform,
2737                    has_dynamic_offset: false,
2738                    min_binding_size: _,
2739                } => (
2740                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2741                    WritableStorage::No,
2742                ),
2743                Bt::Buffer {
2744                    ty: wgt::BufferBindingType::Uniform,
2745                    has_dynamic_offset: true,
2746                    min_binding_size: _,
2747                } => (
2748                    Some(wgt::Features::BUFFER_BINDING_ARRAY),
2749                    WritableStorage::No,
2750                ),
2751                Bt::Buffer {
2752                    ty: wgt::BufferBindingType::Storage { read_only },
2753                    ..
2754                } => (
2755                    Some(
2756                        wgt::Features::BUFFER_BINDING_ARRAY
2757                            | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2758                    ),
2759                    match read_only {
2760                        true => WritableStorage::No,
2761                        false => WritableStorage::Yes,
2762                    },
2763                ),
2764                Bt::Sampler { .. } => (
2765                    Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2766                    WritableStorage::No,
2767                ),
2768                Bt::Texture {
2769                    multisampled: true,
2770                    sample_type: TextureSampleType::Float { filterable: true },
2771                    ..
2772                } => {
2773                    return Err(CreateBindGroupLayoutError::Entry {
2774                        binding: entry.binding,
2775                        error:
2776                            BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
2777                    });
2778                }
2779                Bt::Texture {
2780                    multisampled,
2781                    view_dimension,
2782                    ..
2783                } => {
2784                    if multisampled && view_dimension != TextureViewDimension::D2 {
2785                        return Err(CreateBindGroupLayoutError::Entry {
2786                            binding: entry.binding,
2787                            error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
2788                        });
2789                    }
2790
2791                    (
2792                        Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2793                        WritableStorage::No,
2794                    )
2795                }
2796                Bt::StorageTexture {
2797                    access,
2798                    view_dimension,
2799                    format,
2800                } => {
2801                    use wgt::{StorageTextureAccess as Access, TextureFormatFeatureFlags as Flags};
2802
2803                    match view_dimension {
2804                        TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2805                            return Err(CreateBindGroupLayoutError::Entry {
2806                                binding: entry.binding,
2807                                error: BindGroupLayoutEntryError::StorageTextureCube,
2808                            })
2809                        }
2810                        _ => (),
2811                    }
2812                    match access {
2813                        wgt::StorageTextureAccess::Atomic
2814                            if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
2815                        {
2816                            return Err(CreateBindGroupLayoutError::Entry {
2817                                binding: entry.binding,
2818                                error: BindGroupLayoutEntryError::StorageTextureAtomic,
2819                            });
2820                        }
2821                        _ => (),
2822                    }
2823
2824                    let format_features =
2825                        self.describe_format_features(format).map_err(|error| {
2826                            CreateBindGroupLayoutError::Entry {
2827                                binding: entry.binding,
2828                                error: BindGroupLayoutEntryError::MissingFeatures(error),
2829                            }
2830                        })?;
2831
2832                    let required_feature_flag = match access {
2833                        Access::WriteOnly => Flags::STORAGE_WRITE_ONLY,
2834                        Access::ReadOnly => Flags::STORAGE_READ_ONLY,
2835                        Access::ReadWrite => Flags::STORAGE_READ_WRITE,
2836                        Access::Atomic => Flags::STORAGE_ATOMIC,
2837                    };
2838
2839                    if !format_features.flags.contains(required_feature_flag) {
2840                        return Err(
2841                            CreateBindGroupLayoutError::UnsupportedStorageTextureAccess {
2842                                binding: entry.binding,
2843                                access,
2844                                format,
2845                            },
2846                        );
2847                    }
2848
2849                    (
2850                        Some(
2851                            wgt::Features::TEXTURE_BINDING_ARRAY
2852                                | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2853                        ),
2854                        match access {
2855                            wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
2856                            wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
2857                            wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
2858                            wgt::StorageTextureAccess::Atomic => {
2859                                required_features |= wgt::Features::TEXTURE_ATOMIC;
2860                                WritableStorage::Yes
2861                            }
2862                        },
2863                    )
2864                }
2865                Bt::AccelerationStructure { vertex_return } => {
2866                    self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)
2867                        .map_err(|e| CreateBindGroupLayoutError::Entry {
2868                            binding: entry.binding,
2869                            error: e.into(),
2870                        })?;
2871                    if vertex_return {
2872                        self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
2873                            .map_err(|e| CreateBindGroupLayoutError::Entry {
2874                                binding: entry.binding,
2875                                error: e.into(),
2876                            })?;
2877                    }
2878                    (
2879                        Some(wgt::Features::ACCELERATION_STRUCTURE_BINDING_ARRAY),
2880                        WritableStorage::No,
2881                    )
2882                }
2883                Bt::ExternalTexture => {
2884                    self.require_features(wgt::Features::EXTERNAL_TEXTURE)
2885                        .map_err(|e| CreateBindGroupLayoutError::Entry {
2886                            binding: entry.binding,
2887                            error: e.into(),
2888                        })?;
2889                    (None, WritableStorage::No)
2890                }
2891            };
2892
2893            // Validate the count parameter
2894            if entry.count.is_some() {
2895                required_features |= array_feature
2896                    .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
2897                    .map_err(|error| CreateBindGroupLayoutError::Entry {
2898                        binding: entry.binding,
2899                        error,
2900                    })?;
2901            }
2902
2903            if entry.visibility.contains_unknown_bits() {
2904                return Err(CreateBindGroupLayoutError::InvalidVisibility(
2905                    entry.visibility,
2906                ));
2907            }
2908
2909            if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
2910                if writable_storage == WritableStorage::Yes {
2911                    required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
2912                }
2913                if let Bt::Buffer {
2914                    ty: wgt::BufferBindingType::Storage { .. },
2915                    ..
2916                } = entry.ty
2917                {
2918                    required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
2919                }
2920            }
2921            if writable_storage == WritableStorage::Yes
2922                && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
2923            {
2924                required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
2925            }
2926
2927            self.require_features(required_features)
2928                .map_err(BindGroupLayoutEntryError::MissingFeatures)
2929                .map_err(|error| CreateBindGroupLayoutError::Entry {
2930                    binding: entry.binding,
2931                    error,
2932                })?;
2933            self.require_downlevel_flags(required_downlevel_flags)
2934                .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
2935                .map_err(|error| CreateBindGroupLayoutError::Entry {
2936                    binding: entry.binding,
2937                    error,
2938                })?;
2939        }
2940
2941        let bgl_flags = conv::bind_group_layout_flags(self.features);
2942
2943        let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
2944        let hal_desc = hal::BindGroupLayoutDescriptor {
2945            label: label.to_hal(self.instance_flags),
2946            flags: bgl_flags,
2947            entries: &hal_bindings,
2948        };
2949
2950        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
2951        for entry in entry_map.values() {
2952            count_validator.add_binding(entry);
2953        }
2954        // If a single bind group layout violates limits, the pipeline layout is
2955        // definitely going to violate limits too, lets catch it now.
2956        count_validator
2957            .validate(&self.limits, self.instance_flags)
2958            .map_err(CreateBindGroupLayoutError::TooManyBindings)?;
2959
2960        // Validate that binding arrays don't conflict with dynamic offsets.
2961        count_validator.validate_binding_arrays()?;
2962
2963        let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
2964            .map_err(|e| self.handle_hal_error(e))?;
2965
2966        let bgl = BindGroupLayout {
2967            state: ResourceState::Valid(BindGroupLayoutState {
2968                raw: binding_model::RawBindGroupLayout::Owning(ManuallyDrop::new(raw)),
2969                origin,
2970                binding_count_validator: count_validator,
2971            }),
2972            device: self.clone(),
2973            entries: entry_map,
2974            exclusive_pipeline: OnceCellOrLock::new(),
2975            label: label.to_string(),
2976        };
2977
2978        let bgl = Arc::new(bgl);
2979
2980        Ok(bgl)
2981    }
2982
2983    fn create_buffer_binding<'a>(
2984        &self,
2985        bb: &'a binding_model::ResolvedBufferBinding,
2986        binding: u32,
2987        decl: &wgt::BindGroupLayoutEntry,
2988        buffer_init_actions: &mut Vec<BufferInitTrackerAction>,
2989        dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
2990        late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
2991        used: &mut BindGroupStates,
2992        snatch_guard: &'a SnatchGuard<'a>,
2993    ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer>, CreateBindGroupError> {
2994        use crate::binding_model::CreateBindGroupError as Error;
2995
2996        let (binding_ty, dynamic, min_size) = match decl.ty {
2997            wgt::BindingType::Buffer {
2998                ty,
2999                has_dynamic_offset,
3000                min_binding_size,
3001            } => (ty, has_dynamic_offset, min_binding_size),
3002            _ => {
3003                return Err(Error::WrongBindingType {
3004                    binding,
3005                    actual: decl.ty,
3006                    expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
3007                })
3008            }
3009        };
3010
3011        let (pub_usage, internal_use, range_limit) = match binding_ty {
3012            wgt::BufferBindingType::Uniform => (
3013                wgt::BufferUsages::UNIFORM,
3014                wgt::BufferUses::UNIFORM,
3015                self.limits.max_uniform_buffer_binding_size,
3016            ),
3017            wgt::BufferBindingType::Storage { read_only } => (
3018                wgt::BufferUsages::STORAGE,
3019                if read_only {
3020                    wgt::BufferUses::STORAGE_READ_ONLY
3021                } else {
3022                    wgt::BufferUses::STORAGE_READ_WRITE
3023                },
3024                self.limits.max_storage_buffer_binding_size,
3025            ),
3026        };
3027
3028        let (align, align_limit_name) =
3029            binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
3030        if !bb.offset.is_multiple_of(align as u64) {
3031            return Err(Error::UnalignedBufferOffset(
3032                bb.offset,
3033                align_limit_name,
3034                align,
3035            ));
3036        }
3037
3038        let buffer = &bb.buffer;
3039
3040        used.buffers.insert_single(buffer.clone(), internal_use);
3041
3042        buffer.same_device(self)?;
3043
3044        buffer.check_usage(pub_usage)?;
3045
3046        let req_size = match bb.size.map(wgt::BufferSize::new) {
3047            // Requested a non-zero size
3048            Some(non_zero @ Some(_)) => non_zero,
3049            // Requested size not specified
3050            None => None,
3051            // Requested zero size
3052            Some(None) => return Err(CreateBindGroupError::BindingZeroSize(buffer.error_ident())),
3053        };
3054        let (bb, bind_size) = buffer.binding(bb.offset, req_size, snatch_guard)?;
3055
3056        if matches!(binding_ty, wgt::BufferBindingType::Storage { .. })
3057            && bind_size % u64::from(wgt::STORAGE_BINDING_SIZE_ALIGNMENT) != 0
3058        {
3059            return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
3060                alignment: wgt::STORAGE_BINDING_SIZE_ALIGNMENT,
3061                size: bind_size,
3062            });
3063        }
3064
3065        let bind_end = bb.offset + bind_size;
3066
3067        if bind_size > range_limit {
3068            return Err(Error::BufferRangeTooLarge {
3069                binding,
3070                given: bind_size,
3071                limit: range_limit,
3072            });
3073        }
3074
3075        // Record binding info for validating dynamic offsets
3076        if dynamic {
3077            dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
3078                binding_idx: binding,
3079                buffer_size: buffer.size,
3080                binding_range: bb.offset..bind_end,
3081                maximum_dynamic_offset: buffer.size - bind_end,
3082                binding_type: binding_ty,
3083            });
3084        }
3085
3086        if let Some(non_zero) = min_size {
3087            let min_size = non_zero.get();
3088            if min_size > bind_size {
3089                return Err(Error::BindingSizeTooSmall {
3090                    buffer: buffer.error_ident(),
3091                    actual: bind_size,
3092                    min: min_size,
3093                });
3094            }
3095        } else {
3096            let late_size = wgt::BufferSize::new(bind_size)
3097                .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
3098            late_buffer_binding_sizes.insert(binding, late_size);
3099        }
3100
3101        // This was checked against the device's alignment requirements above,
3102        // which should always be a multiple of `COPY_BUFFER_ALIGNMENT`.
3103        assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
3104
3105        let init_range = if dynamic {
3106            // We don't know what part of the buffer will be bound, so require that it
3107            // is fully initialized.
3108            0..buffer.size
3109        } else {
3110            // `wgpu_hal` only restricts shader access to bound buffer regions with
3111            // a certain resolution. For the sake of lazy initialization, round up
3112            // the size of the bound range to reflect how much of the buffer is
3113            // actually going to be visible to the shader.
3114            let bounds_check_alignment = binding_model::buffer_binding_type_bounds_check_alignment(
3115                &self.alignments,
3116                binding_ty,
3117            );
3118            let visible_size = align_to(bind_size, bounds_check_alignment);
3119
3120            bb.offset..bb.offset + visible_size
3121        };
3122
3123        buffer_init_actions.extend(buffer.initialization_status.read().create_action(
3124            buffer,
3125            init_range,
3126            MemoryInitKind::NeedsInitializedMemory,
3127        ));
3128
3129        Ok(bb)
3130    }
3131
3132    fn create_sampler_binding<'a>(
3133        &self,
3134        used: &mut BindGroupStates,
3135        binding: u32,
3136        decl: &wgt::BindGroupLayoutEntry,
3137        sampler: &'a Arc<Sampler>,
3138    ) -> Result<&'a dyn hal::DynSampler, CreateBindGroupError> {
3139        use crate::binding_model::CreateBindGroupError as Error;
3140
3141        used.samplers.insert_single(sampler.clone());
3142
3143        sampler.same_device(self)?;
3144
3145        match decl.ty {
3146            wgt::BindingType::Sampler(ty) => {
3147                let (allowed_filtering, allowed_comparison) = match ty {
3148                    wgt::SamplerBindingType::Filtering => (None, false),
3149                    wgt::SamplerBindingType::NonFiltering => (Some(false), false),
3150                    wgt::SamplerBindingType::Comparison => (None, true),
3151                };
3152                if let Some(allowed_filtering) = allowed_filtering {
3153                    if allowed_filtering != sampler.filtering {
3154                        return Err(Error::WrongSamplerFiltering {
3155                            binding,
3156                            layout_flt: allowed_filtering,
3157                            sampler_flt: sampler.filtering,
3158                        });
3159                    }
3160                }
3161                if allowed_comparison != sampler.comparison {
3162                    return Err(Error::WrongSamplerComparison {
3163                        binding,
3164                        layout_cmp: allowed_comparison,
3165                        sampler_cmp: sampler.comparison,
3166                    });
3167                }
3168            }
3169            _ => {
3170                return Err(Error::WrongBindingType {
3171                    binding,
3172                    actual: decl.ty,
3173                    expected: "Sampler",
3174                })
3175            }
3176        }
3177
3178        Ok(sampler.raw())
3179    }
3180
3181    fn create_texture_binding<'a>(
3182        &self,
3183        binding: u32,
3184        decl: &wgt::BindGroupLayoutEntry,
3185        view: &'a Arc<TextureView>,
3186        used: &mut BindGroupStates,
3187        texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3188        snatch_guard: &'a SnatchGuard<'a>,
3189    ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, CreateBindGroupError> {
3190        view.same_device(self)?;
3191
3192        let internal_use = self.texture_use_parameters(
3193            binding,
3194            decl,
3195            view,
3196            "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
3197        )?;
3198
3199        used.views.insert_single(view.clone(), internal_use);
3200
3201        let texture = &view.parent;
3202
3203        texture_init_actions.push(TextureInitTrackerAction {
3204            texture: texture.clone(),
3205            range: TextureInitRange {
3206                mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
3207                layer_range: view
3208                    .desc
3209                    .range
3210                    .layer_range(texture.desc.array_layer_count()),
3211            },
3212            kind: MemoryInitKind::NeedsInitializedMemory,
3213        });
3214
3215        Ok(hal::TextureBinding {
3216            view: view.try_raw(snatch_guard)?,
3217            usage: internal_use,
3218        })
3219    }
3220
3221    fn create_tlas_binding<'a>(
3222        self: &Arc<Self>,
3223        used: &mut BindGroupStates,
3224        binding: u32,
3225        decl: &wgt::BindGroupLayoutEntry,
3226        tlas: &'a Arc<Tlas>,
3227        snatch_guard: &'a SnatchGuard<'a>,
3228    ) -> Result<&'a dyn hal::DynAccelerationStructure, CreateBindGroupError> {
3229        use crate::binding_model::CreateBindGroupError as Error;
3230
3231        used.acceleration_structures.insert_single(tlas.clone());
3232
3233        tlas.same_device(self)?;
3234
3235        match decl.ty {
3236            wgt::BindingType::AccelerationStructure { vertex_return } => {
3237                if vertex_return
3238                    && !tlas.flags.contains(
3239                        wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
3240                    )
3241                {
3242                    return Err(Error::MissingTLASVertexReturn { binding });
3243                }
3244            }
3245            _ => {
3246                return Err(Error::WrongBindingType {
3247                    binding,
3248                    actual: decl.ty,
3249                    expected: "Tlas",
3250                });
3251            }
3252        }
3253
3254        Ok(tlas.try_raw(snatch_guard)?)
3255    }
3256
3257    fn create_external_texture_binding<'a>(
3258        &'a self,
3259        binding: u32,
3260        decl: &wgt::BindGroupLayoutEntry,
3261        external_texture: &'a Arc<ExternalTexture>,
3262        used: &mut BindGroupStates,
3263        snatch_guard: &'a SnatchGuard,
3264    ) -> Result<
3265        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3266        CreateBindGroupError,
3267    > {
3268        use crate::binding_model::CreateBindGroupError as Error;
3269
3270        external_texture.same_device(self)?;
3271
3272        used.external_textures
3273            .insert_single(external_texture.clone());
3274
3275        match decl.ty {
3276            wgt::BindingType::ExternalTexture => {}
3277            _ => {
3278                return Err(Error::WrongBindingType {
3279                    binding,
3280                    actual: decl.ty,
3281                    expected: "ExternalTexture",
3282                });
3283            }
3284        }
3285
3286        let planes = (0..3)
3287            .map(|i| {
3288                // We always need 3 bindings. If we have fewer than 3 planes
3289                // just bind plane 0 multiple times. The shader will only
3290                // sample from valid planes anyway.
3291                let plane = external_texture
3292                    .planes
3293                    .get(i)
3294                    .unwrap_or(&external_texture.planes[0]);
3295                let internal_use = wgt::TextureUses::RESOURCE;
3296                used.views.insert_single(plane.clone(), internal_use);
3297                let view = plane.try_raw(snatch_guard)?;
3298                Ok(hal::TextureBinding {
3299                    view,
3300                    usage: internal_use,
3301                })
3302            })
3303            // We can remove this intermediate Vec by using
3304            // array::try_from_fn() above, once it stabilizes.
3305            .collect::<Result<Vec<_>, Error>>()?;
3306        let planes = planes.try_into().unwrap();
3307
3308        used.buffers
3309            .insert_single(external_texture.params.clone(), wgt::BufferUses::UNIFORM);
3310        let params = external_texture.params.binding(0, None, snatch_guard)?.0;
3311
3312        Ok(hal::ExternalTextureBinding { planes, params })
3313    }
3314
3315    fn create_external_texture_binding_from_view<'a>(
3316        &'a self,
3317        binding: u32,
3318        decl: &wgt::BindGroupLayoutEntry,
3319        view: &'a Arc<TextureView>,
3320        used: &mut BindGroupStates,
3321        snatch_guard: &'a SnatchGuard,
3322    ) -> Result<
3323        hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3324        CreateBindGroupError,
3325    > {
3326        use crate::binding_model::CreateBindGroupError as Error;
3327
3328        view.same_device(self)?;
3329
3330        let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
3331        used.views.insert_single(view.clone(), internal_use);
3332
3333        match decl.ty {
3334            wgt::BindingType::ExternalTexture => {}
3335            _ => {
3336                return Err(Error::WrongBindingType {
3337                    binding,
3338                    actual: decl.ty,
3339                    expected: "ExternalTexture",
3340                });
3341            }
3342        }
3343
3344        // We need 3 bindings, so just repeat the same texture view 3 times.
3345        let planes = [
3346            hal::TextureBinding {
3347                view: view.try_raw(snatch_guard)?,
3348                usage: internal_use,
3349            },
3350            hal::TextureBinding {
3351                view: view.try_raw(snatch_guard)?,
3352                usage: internal_use,
3353            },
3354            hal::TextureBinding {
3355                view: view.try_raw(snatch_guard)?,
3356                usage: internal_use,
3357            },
3358        ];
3359        let params = hal::BufferBinding::new_unchecked(
3360            self.default_external_texture_params_buffer.as_ref(),
3361            0,
3362            None,
3363        );
3364
3365        Ok(hal::ExternalTextureBinding { planes, params })
3366    }
3367
3368    // This function expects the provided bind group layout to be resolved
3369    // (not passing a duplicate) beforehand.
3370    pub fn create_bind_group(
3371        self: &Arc<Self>,
3372        desc: binding_model::ResolvedBindGroupDescriptor,
3373    ) -> Result<Arc<BindGroup>, CreateBindGroupError> {
3374        use crate::binding_model::{CreateBindGroupError as Error, ResolvedBindingResource as Br};
3375
3376        let layout = desc.layout;
3377
3378        self.check_is_valid()?;
3379        layout.same_device(self)?;
3380        layout.check_is_valid()?;
3381
3382        {
3383            // Check that the number of entries in the descriptor matches
3384            // the number of entries in the layout.
3385            let actual = desc.entries.len();
3386            let expected = layout.entries.len();
3387            if actual != expected {
3388                return Err(Error::BindingsNumMismatch { expected, actual });
3389            }
3390        }
3391
3392        // TODO: arrayvec/smallvec, or re-use allocations
3393        // Record binding info for dynamic offset validation
3394        let mut dynamic_binding_info = Vec::new();
3395        // Map of binding -> shader reflected size
3396        //Note: we can't collect into a vector right away because
3397        // it needs to be in BGL iteration order, not BG entry order.
3398        let mut late_buffer_binding_sizes = FastHashMap::default();
3399        // fill out the descriptors
3400        let mut used = BindGroupStates::new();
3401
3402        let mut buffer_init_actions = Vec::new();
3403        let mut texture_init_actions = Vec::new();
3404        let mut hal_entries = Vec::with_capacity(desc.entries.len());
3405        let mut hal_buffers = Vec::new();
3406        let mut hal_samplers = Vec::new();
3407        let mut hal_textures = Vec::new();
3408        let mut hal_tlas_s = Vec::new();
3409        let mut hal_external_textures = Vec::new();
3410        let snatch_guard = self.snatchable_lock.read();
3411        for entry in desc.entries.iter() {
3412            let binding = entry.binding;
3413            // Find the corresponding declaration in the layout
3414            let decl = layout
3415                .entries
3416                .get(binding)
3417                .ok_or(Error::MissingBindingDeclaration(binding))?;
3418            let (res_index, count) = match entry.resource {
3419                Br::Buffer(ref bb) => {
3420                    let bb = self.create_buffer_binding(
3421                        bb,
3422                        binding,
3423                        decl,
3424                        &mut buffer_init_actions,
3425                        &mut dynamic_binding_info,
3426                        &mut late_buffer_binding_sizes,
3427                        &mut used,
3428                        &snatch_guard,
3429                    )?;
3430
3431                    let res_index = hal_buffers.len();
3432                    hal_buffers.push(bb);
3433                    (res_index, 1)
3434                }
3435                Br::BufferArray(ref bindings_array) => {
3436                    let num_bindings = bindings_array.len();
3437                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3438
3439                    let res_index = hal_buffers.len();
3440                    for bb in bindings_array.iter() {
3441                        let bb = self.create_buffer_binding(
3442                            bb,
3443                            binding,
3444                            decl,
3445                            &mut buffer_init_actions,
3446                            &mut dynamic_binding_info,
3447                            &mut late_buffer_binding_sizes,
3448                            &mut used,
3449                            &snatch_guard,
3450                        )?;
3451                        hal_buffers.push(bb);
3452                    }
3453                    (res_index, num_bindings)
3454                }
3455                Br::Sampler(ref sampler) => {
3456                    let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3457
3458                    let res_index = hal_samplers.len();
3459                    hal_samplers.push(sampler);
3460                    (res_index, 1)
3461                }
3462                Br::SamplerArray(ref samplers) => {
3463                    let num_bindings = samplers.len();
3464                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3465
3466                    let res_index = hal_samplers.len();
3467                    for sampler in samplers.iter() {
3468                        let sampler =
3469                            self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3470
3471                        hal_samplers.push(sampler);
3472                    }
3473
3474                    (res_index, num_bindings)
3475                }
3476                Br::TextureView(ref view) => match decl.ty {
3477                    wgt::BindingType::ExternalTexture => {
3478                        let et = self.create_external_texture_binding_from_view(
3479                            binding,
3480                            decl,
3481                            view,
3482                            &mut used,
3483                            &snatch_guard,
3484                        )?;
3485                        let res_index = hal_external_textures.len();
3486                        hal_external_textures.push(et);
3487                        (res_index, 1)
3488                    }
3489                    _ => {
3490                        let tb = self.create_texture_binding(
3491                            binding,
3492                            decl,
3493                            view,
3494                            &mut used,
3495                            &mut texture_init_actions,
3496                            &snatch_guard,
3497                        )?;
3498                        let res_index = hal_textures.len();
3499                        hal_textures.push(tb);
3500                        (res_index, 1)
3501                    }
3502                },
3503                Br::TextureViewArray(ref views) => {
3504                    let num_bindings = views.len();
3505                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3506
3507                    let res_index = hal_textures.len();
3508                    for view in views.iter() {
3509                        let tb = self.create_texture_binding(
3510                            binding,
3511                            decl,
3512                            view,
3513                            &mut used,
3514                            &mut texture_init_actions,
3515                            &snatch_guard,
3516                        )?;
3517
3518                        hal_textures.push(tb);
3519                    }
3520
3521                    (res_index, num_bindings)
3522                }
3523                Br::AccelerationStructure(ref tlas) => {
3524                    let tlas =
3525                        self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
3526                    let res_index = hal_tlas_s.len();
3527                    hal_tlas_s.push(tlas);
3528                    (res_index, 1)
3529                }
3530                Br::AccelerationStructureArray(ref tlas_array) => {
3531                    // Feature validation for TLAS binding arrays happens at bind group layout
3532                    // creation time (mirroring other binding-array resource types). By the time we
3533                    // get here, `decl.count` has already been validated against device features.
3534                    let num_bindings = tlas_array.len();
3535                    Self::check_array_binding(self.features, decl.count, num_bindings)?;
3536
3537                    let res_index = hal_tlas_s.len();
3538                    for tlas in tlas_array.iter() {
3539                        let tlas = self.create_tlas_binding(
3540                            &mut used,
3541                            binding,
3542                            decl,
3543                            tlas,
3544                            &snatch_guard,
3545                        )?;
3546                        hal_tlas_s.push(tlas);
3547                    }
3548                    (res_index, num_bindings)
3549                }
3550                Br::ExternalTexture(ref et) => {
3551                    let et = self.create_external_texture_binding(
3552                        binding,
3553                        decl,
3554                        et,
3555                        &mut used,
3556                        &snatch_guard,
3557                    )?;
3558                    let res_index = hal_external_textures.len();
3559                    hal_external_textures.push(et);
3560                    (res_index, 1)
3561                }
3562            };
3563
3564            hal_entries.push(hal::BindGroupEntry {
3565                binding,
3566                resource_index: res_index as u32,
3567                count: count as u32,
3568            });
3569        }
3570
3571        used.optimize();
3572
3573        hal_entries.sort_by_key(|entry| entry.binding);
3574        for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
3575            if a.binding == b.binding {
3576                return Err(Error::DuplicateBinding(a.binding));
3577            }
3578        }
3579
3580        dynamic_binding_info.sort_by_key(|i| i.binding_idx);
3581
3582        let hal_desc = hal::BindGroupDescriptor {
3583            label: desc.label.to_hal(self.instance_flags),
3584            layout: layout.try_raw()?,
3585            entries: &hal_entries,
3586            buffers: &hal_buffers,
3587            samplers: &hal_samplers,
3588            textures: &hal_textures,
3589            acceleration_structures: &hal_tlas_s,
3590            external_textures: &hal_external_textures,
3591        };
3592        let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
3593            .map_err(|e| self.handle_hal_error(e))?;
3594
3595        // collect in the order of BGL iteration
3596        let late_buffer_binding_infos = layout
3597            .entries
3598            .indices()
3599            .flat_map(|binding| {
3600                let size = late_buffer_binding_sizes.get(&binding).cloned()?;
3601                Some(BindGroupLateBufferBindingInfo {
3602                    binding_index: binding,
3603                    size,
3604                })
3605            })
3606            .collect();
3607
3608        let bind_group = BindGroup {
3609            raw: Snatchable::new(raw),
3610            device: self.clone(),
3611            layout,
3612            label: desc.label.to_string(),
3613            tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
3614            used,
3615            buffer_init_actions,
3616            texture_init_actions,
3617            dynamic_binding_info,
3618            late_buffer_binding_infos,
3619        };
3620
3621        let bind_group = Arc::new(bind_group);
3622
3623        let weak_ref = Arc::downgrade(&bind_group);
3624        for texture in bind_group.used.views.used_textures() {
3625            let mut bind_groups = texture.bind_groups.lock();
3626            bind_groups.push(weak_ref.clone());
3627        }
3628        for buffer in bind_group.used.buffers.used_resources() {
3629            let mut bind_groups = buffer.bind_groups.lock();
3630            bind_groups.push(weak_ref.clone());
3631        }
3632
3633        Ok(bind_group)
3634    }
3635
3636    fn check_array_binding(
3637        features: wgt::Features,
3638        count: Option<NonZeroU32>,
3639        num_bindings: usize,
3640    ) -> Result<(), CreateBindGroupError> {
3641        use super::binding_model::CreateBindGroupError as Error;
3642
3643        if let Some(count) = count {
3644            let count = count.get() as usize;
3645            if count < num_bindings {
3646                return Err(Error::BindingArrayPartialLengthMismatch {
3647                    actual: num_bindings,
3648                    expected: count,
3649                });
3650            }
3651            if count != num_bindings
3652                && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
3653            {
3654                return Err(Error::BindingArrayLengthMismatch {
3655                    actual: num_bindings,
3656                    expected: count,
3657                });
3658            }
3659            if num_bindings == 0 {
3660                return Err(Error::BindingArrayZeroLength);
3661            }
3662        } else {
3663            return Err(Error::SingleBindingExpected);
3664        };
3665
3666        Ok(())
3667    }
3668
3669    fn texture_use_parameters(
3670        &self,
3671        binding: u32,
3672        decl: &wgt::BindGroupLayoutEntry,
3673        view: &TextureView,
3674        expected: &'static str,
3675    ) -> Result<wgt::TextureUses, CreateBindGroupError> {
3676        use crate::binding_model::CreateBindGroupError as Error;
3677        if view
3678            .desc
3679            .aspects()
3680            .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
3681        {
3682            return Err(Error::DepthStencilAspect);
3683        }
3684        match decl.ty {
3685            wgt::BindingType::Texture {
3686                sample_type,
3687                view_dimension,
3688                multisampled,
3689            } => {
3690                use wgt::TextureSampleType as Tst;
3691                if multisampled != (view.samples != 1) {
3692                    return Err(Error::InvalidTextureMultisample {
3693                        binding,
3694                        layout_multisampled: multisampled,
3695                        view_samples: view.samples,
3696                    });
3697                }
3698                let compat_sample_type = view
3699                    .desc
3700                    .format
3701                    .sample_type(Some(view.desc.range.aspect), Some(self.features))
3702                    .unwrap();
3703                match (sample_type, compat_sample_type) {
3704                    (Tst::Uint, Tst::Uint) |
3705                        (Tst::Sint, Tst::Sint) |
3706                        (Tst::Depth, Tst::Depth) |
3707                        // if we expect non-filterable, accept anything float
3708                        (Tst::Float { filterable: false }, Tst::Float { .. }) |
3709                        // if we expect filterable, require it
3710                        (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
3711                        // if we expect non-filterable, also accept depth
3712                        (Tst::Float { filterable: false }, Tst::Depth) => {}
3713                    // if we expect filterable, also accept Float that is defined as
3714                    // unfilterable if filterable feature is explicitly enabled (only hit
3715                    // if wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES is
3716                    // enabled)
3717                    (Tst::Float { filterable: true }, Tst::Float { .. })
3718                        if view.format_features.flags
3719                            .contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
3720                    _ => {
3721                        return Err(Error::InvalidTextureSampleType {
3722                            binding,
3723                            layout_sample_type: sample_type,
3724                            view_format: view.desc.format,
3725                            view_sample_type: compat_sample_type,
3726                        })
3727                    }
3728                }
3729                if view_dimension != view.desc.dimension {
3730                    return Err(Error::InvalidTextureDimension {
3731                        binding,
3732                        layout_dimension: view_dimension,
3733                        view_dimension: view.desc.dimension,
3734                    });
3735                }
3736                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3737                Ok(wgt::TextureUses::RESOURCE)
3738            }
3739            wgt::BindingType::StorageTexture {
3740                access,
3741                format,
3742                view_dimension,
3743            } => {
3744                if format != view.desc.format {
3745                    return Err(Error::InvalidStorageTextureFormat {
3746                        binding,
3747                        layout_format: format,
3748                        view_format: view.desc.format,
3749                    });
3750                }
3751                if view_dimension != view.desc.dimension {
3752                    return Err(Error::InvalidTextureDimension {
3753                        binding,
3754                        layout_dimension: view_dimension,
3755                        view_dimension: view.desc.dimension,
3756                    });
3757                }
3758
3759                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3760                if mip_level_count != 1 {
3761                    return Err(Error::InvalidStorageTextureMipLevelCount {
3762                        binding,
3763                        mip_level_count,
3764                    });
3765                }
3766
3767                view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
3768
3769                Ok(match access {
3770                    wgt::StorageTextureAccess::ReadOnly => wgt::TextureUses::STORAGE_READ_ONLY,
3771                    wgt::StorageTextureAccess::WriteOnly => wgt::TextureUses::STORAGE_WRITE_ONLY,
3772                    wgt::StorageTextureAccess::ReadWrite => wgt::TextureUses::STORAGE_READ_WRITE,
3773                    wgt::StorageTextureAccess::Atomic => wgt::TextureUses::STORAGE_ATOMIC,
3774                })
3775            }
3776            wgt::BindingType::ExternalTexture => {
3777                if view.desc.dimension != TextureViewDimension::D2 {
3778                    return Err(Error::InvalidTextureDimension {
3779                        binding,
3780                        layout_dimension: TextureViewDimension::D2,
3781                        view_dimension: view.desc.dimension,
3782                    });
3783                }
3784                let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3785                if mip_level_count != 1 {
3786                    return Err(Error::InvalidExternalTextureMipLevelCount {
3787                        binding,
3788                        mip_level_count,
3789                    });
3790                }
3791                if view.desc.format != TextureFormat::Rgba8Unorm
3792                    && view.desc.format != TextureFormat::Bgra8Unorm
3793                    && view.desc.format != TextureFormat::Rgba16Float
3794                {
3795                    return Err(Error::InvalidExternalTextureFormat {
3796                        binding,
3797                        format: view.desc.format,
3798                    });
3799                }
3800                if view.samples != 1 {
3801                    return Err(Error::InvalidTextureMultisample {
3802                        binding,
3803                        layout_multisampled: false,
3804                        view_samples: view.samples,
3805                    });
3806                }
3807
3808                view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3809                Ok(wgt::TextureUses::RESOURCE)
3810            }
3811            _ => Err(Error::WrongBindingType {
3812                binding,
3813                actual: decl.ty,
3814                expected,
3815            }),
3816        }
3817    }
3818
3819    pub fn create_pipeline_layout(
3820        self: &Arc<Self>,
3821        desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3822    ) -> (
3823        Arc<binding_model::PipelineLayout>,
3824        Option<binding_model::CreatePipelineLayoutError>,
3825    ) {
3826        let (layout, error) = match self.create_pipeline_layout_impl(desc, false) {
3827            Ok(layout) => (layout, None),
3828            Err(e) => (
3829                binding_model::PipelineLayout::invalid(Arc::clone(self), desc.label.to_string()),
3830                Some(e),
3831            ),
3832        };
3833        #[cfg(feature = "trace")]
3834        if let Some(ref mut trace) = *self.trace.lock() {
3835            use crate::device::trace::IntoTrace;
3836            trace.add(trace::Action::CreatePipelineLayout(
3837                layout.to_trace(),
3838                desc.to_trace(),
3839            ));
3840        }
3841        api_log!(
3842            "Device::create_pipeline_layout -> {:?}",
3843            Arc::as_ptr(&layout)
3844        );
3845        (layout, error)
3846    }
3847
3848    fn create_pipeline_layout_impl(
3849        self: &Arc<Self>,
3850        desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3851        ignore_exclusive_pipeline_check: bool,
3852    ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
3853        use crate::binding_model::CreatePipelineLayoutError as Error;
3854
3855        self.check_is_valid()?;
3856
3857        let bind_group_layouts_count = desc.bind_group_layouts.len();
3858        let device_max_bind_groups = self.limits.max_bind_groups as usize;
3859        if bind_group_layouts_count > device_max_bind_groups {
3860            return Err(Error::TooManyGroups {
3861                actual: bind_group_layouts_count,
3862                max: device_max_bind_groups,
3863            });
3864        }
3865
3866        if desc.immediate_size != 0 {
3867            self.require_features(wgt::Features::IMMEDIATES)?;
3868        }
3869        if self.limits.max_immediate_size < desc.immediate_size {
3870            return Err(Error::ImmediateRangeTooLarge {
3871                size: desc.immediate_size,
3872                max: self.limits.max_immediate_size,
3873            });
3874        }
3875        if !desc
3876            .immediate_size
3877            .is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT)
3878        {
3879            return Err(Error::MisalignedImmediateSize {
3880                size: desc.immediate_size,
3881            });
3882        }
3883
3884        let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3885
3886        for (index, bgl) in desc.bind_group_layouts.iter().enumerate() {
3887            let Some(bgl) = bgl else {
3888                continue;
3889            };
3890
3891            bgl.same_device(self)?;
3892
3893            if !ignore_exclusive_pipeline_check {
3894                let exclusive_pipeline = bgl.exclusive_pipeline.get().unwrap();
3895                if !matches!(exclusive_pipeline, binding_model::ExclusivePipeline::None) {
3896                    return Err(Error::BglHasExclusivePipeline {
3897                        index,
3898                        pipeline: alloc::format!("{exclusive_pipeline}"),
3899                    });
3900                }
3901            }
3902
3903            count_validator.merge(&bgl.state()?.binding_count_validator);
3904        }
3905
3906        count_validator
3907            .validate(&self.limits, self.instance_flags)
3908            .map_err(Error::TooManyBindings)?;
3909
3910        let buffers_and_acceleration_structures_in_vertex_stage =
3911            count_validator.buffers_and_acceleration_structures_in_vertex_stage();
3912
3913        let get_bgl_iter = || {
3914            desc.bind_group_layouts
3915                .iter()
3916                .map(|bgl| bgl.as_ref().filter(|bgl| !bgl.entries.is_empty()))
3917        };
3918
3919        let bind_group_layouts = get_bgl_iter()
3920            .map(|bgl| bgl.cloned())
3921            .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3922
3923        let raw_bind_group_layouts = get_bgl_iter()
3924            .map(|bgl| bgl.map(|bgl| bgl.try_raw()).transpose())
3925            .collect::<Result<ArrayVec<_, { hal::MAX_BIND_GROUPS }>, _>>()?;
3926
3927        let additional_flags = if self.indirect_validation.is_some() {
3928            hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
3929        } else {
3930            hal::PipelineLayoutFlags::empty()
3931        };
3932
3933        let hal_desc = hal::PipelineLayoutDescriptor {
3934            label: desc.label.to_hal(self.instance_flags),
3935            flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
3936                | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
3937                | additional_flags,
3938            bind_group_layouts: &raw_bind_group_layouts,
3939            immediate_size: desc.immediate_size,
3940        };
3941
3942        let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
3943            .map_err(|e| self.handle_hal_error(e))?;
3944
3945        drop(raw_bind_group_layouts);
3946
3947        let layout = binding_model::PipelineLayout {
3948            raw: ResourceState::Valid(raw),
3949            device: self.clone(),
3950            label: desc.label.to_string(),
3951            bind_group_layouts,
3952            immediate_size: desc.immediate_size,
3953            buffers_and_acceleration_structures_in_vertex_stage,
3954        };
3955
3956        let layout = Arc::new(layout);
3957
3958        Ok(layout)
3959    }
3960
3961    fn create_derived_pipeline_layout(
3962        self: &Arc<Self>,
3963        mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
3964        immediate_size: u32,
3965    ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
3966        while derived_group_layouts
3967            .last()
3968            .is_some_and(|map| map.is_empty())
3969        {
3970            derived_group_layouts.pop();
3971        }
3972
3973        let mut unique_bind_group_layouts = FastHashMap::default();
3974
3975        let bind_group_layouts = derived_group_layouts
3976            .into_iter()
3977            .map(|mut bgl_entry_map| {
3978                if bgl_entry_map.is_empty() {
3979                    return Ok(None);
3980                }
3981
3982                bgl_entry_map.sort();
3983                match unique_bind_group_layouts.entry(bgl_entry_map) {
3984                    hashbrown::hash_map::Entry::Occupied(v) => Ok(Some(Arc::clone(v.get()))),
3985                    hashbrown::hash_map::Entry::Vacant(e) => {
3986                        match self.create_bind_group_layout_impl(
3987                            &None,
3988                            e.key().clone(),
3989                            bgl::Origin::Derived,
3990                        ) {
3991                            Ok(bgl) => {
3992                                e.insert(bgl.clone());
3993                                Ok(Some(bgl))
3994                            }
3995                            Err(e) => Err(e),
3996                        }
3997                    }
3998                }
3999            })
4000            .collect::<Result<Vec<_>, _>>()?;
4001
4002        let layout_desc = binding_model::ResolvedPipelineLayoutDescriptor {
4003            label: None,
4004            bind_group_layouts: Cow::Owned(bind_group_layouts),
4005            immediate_size,
4006        };
4007
4008        let layout = self.create_pipeline_layout_impl(&layout_desc, true)?;
4009        Ok(layout)
4010    }
4011
4012    pub fn create_compute_pipeline(
4013        self: &Arc<Self>,
4014        desc: pipeline::ResolvedComputePipelineDescriptor,
4015    ) -> (
4016        Arc<pipeline::ComputePipeline>,
4017        Option<pipeline::CreateComputePipelineError>,
4018    ) {
4019        let (compute_pipeline, error) = match self.create_compute_pipeline_inner(desc.clone()) {
4020            Ok(compute_pipeline) => (compute_pipeline, None),
4021            Err(error) => (
4022                pipeline::ComputePipeline::invalid(self.clone(), desc.label.to_string()),
4023                Some(error),
4024            ),
4025        };
4026        #[cfg(feature = "trace")]
4027        if let Some(ref mut trace) = *self.trace.lock() {
4028            use crate::device::trace;
4029            use crate::device::trace::IntoTrace;
4030            trace.add(trace::Action::CreateComputePipeline {
4031                id: compute_pipeline.to_trace(),
4032                desc: desc.to_trace(),
4033            });
4034        }
4035        (compute_pipeline, error)
4036    }
4037
4038    pub fn create_compute_pipeline_inner(
4039        self: &Arc<Self>,
4040        desc: pipeline::ResolvedComputePipelineDescriptor,
4041    ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreateComputePipelineError> {
4042        self.check_is_valid()?;
4043
4044        self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
4045
4046        let shader_module = desc.stage.module;
4047
4048        shader_module.same_device(self)?;
4049
4050        let is_auto_layout = desc.layout.is_none();
4051
4052        // Get the pipeline layout from the desc if it is provided.
4053        let pipeline_layout = match desc.layout {
4054            Some(pipeline_layout) => {
4055                pipeline_layout.same_device(self)?;
4056                pipeline_layout.check_valid()?;
4057                Some(pipeline_layout)
4058            }
4059            None => None,
4060        };
4061
4062        let mut binding_layout_source = match pipeline_layout {
4063            Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4064            None => validation::BindingLayoutSource::new_derived(&self.limits),
4065        };
4066        let mut shader_binding_sizes = FastHashMap::default();
4067        let io = validation::StageIo::default();
4068
4069        let final_entry_point_name;
4070
4071        {
4072            let stage = validation::ShaderStageForValidation::Compute;
4073
4074            final_entry_point_name = shader_module.finalize_entry_point_name(
4075                stage.to_naga(),
4076                desc.stage.entry_point.as_ref().map(|ep| ep.as_ref()),
4077            )?;
4078
4079            if let Some(interface) = shader_module.interface.interface() {
4080                let _ = interface.check_stage(
4081                    &mut binding_layout_source,
4082                    &mut shader_binding_sizes,
4083                    &final_entry_point_name,
4084                    stage,
4085                    io,
4086                    None,
4087                )?;
4088            }
4089        }
4090
4091        let pipeline_layout = match binding_layout_source {
4092            validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4093            validation::BindingLayoutSource::Derived(entries) => {
4094                let immediate_size = shader_module
4095                    .interface
4096                    .interface()
4097                    .map_or(0, |i| i.immediate_size);
4098                self.create_derived_pipeline_layout(entries, immediate_size)?
4099            }
4100        };
4101
4102        let late_sized_buffer_groups =
4103            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4104
4105        let cache = match desc.cache {
4106            Some(cache) => {
4107                cache.same_device(self)?;
4108                Some(cache)
4109            }
4110            None => None,
4111        };
4112
4113        let pipeline_desc = hal::ComputePipelineDescriptor {
4114            label: desc.label.to_hal(self.instance_flags),
4115            layout: pipeline_layout.raw()?,
4116            stage: hal::ProgrammableStage {
4117                module: shader_module.raw(),
4118                entry_point: final_entry_point_name.as_ref(),
4119                constants: &desc.stage.constants,
4120                zero_initialize_workgroup_memory: desc.stage.zero_initialize_workgroup_memory,
4121            },
4122            cache: cache.as_ref().map(|it| it.raw()),
4123        };
4124
4125        let raw =
4126            unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }.map_err(
4127                |err| match err {
4128                    hal::PipelineError::Device(error) => {
4129                        pipeline::CreateComputePipelineError::Device(self.handle_hal_error(error))
4130                    }
4131                    hal::PipelineError::Linkage(_stages, msg) => {
4132                        pipeline::CreateComputePipelineError::Internal(msg)
4133                    }
4134                    hal::PipelineError::EntryPoint(_stage) => {
4135                        pipeline::CreateComputePipelineError::Internal(
4136                            ENTRYPOINT_FAILURE_ERROR.to_string(),
4137                        )
4138                    }
4139                    hal::PipelineError::PipelineConstants(_stages, msg) => {
4140                        pipeline::CreateComputePipelineError::PipelineConstants(msg)
4141                    }
4142                },
4143            )?;
4144
4145        let immediate_slots_required =
4146            shader_module
4147                .interface
4148                .interface()
4149                .map_or(Default::default(), |iface| {
4150                    iface.immediate_slots_required(
4151                        naga::ShaderStage::Compute,
4152                        &final_entry_point_name,
4153                    )
4154                });
4155
4156        let pipeline = pipeline::ComputePipeline {
4157            state: ResourceState::Valid(pipeline::ComputePipelineState {
4158                raw: ManuallyDrop::new(raw),
4159                layout: pipeline_layout.clone(),
4160                _shader_module: shader_module,
4161            }),
4162            device: self.clone(),
4163            late_sized_buffer_groups,
4164            immediate_slots_required,
4165            label: desc.label.to_string(),
4166            tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
4167        };
4168
4169        let pipeline = Arc::new(pipeline);
4170
4171        if is_auto_layout {
4172            for bgl in pipeline_layout.bind_group_layouts.iter() {
4173                let Some(bgl) = bgl else {
4174                    continue;
4175                };
4176
4177                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the
4178                // result.
4179                let _ = bgl.exclusive_pipeline.set((&pipeline).into());
4180            }
4181        }
4182
4183        Ok(pipeline)
4184    }
4185
4186    pub fn create_render_pipeline(
4187        self: &Arc<Self>,
4188        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4189    ) -> (
4190        Arc<pipeline::RenderPipeline>,
4191        Option<pipeline::CreateRenderPipelineError>,
4192    ) {
4193        let (render_pipeline, error) = match self.create_render_pipeline_inner(desc.clone()) {
4194            Ok(pipeline) => (pipeline, None),
4195            Err(e) => (
4196                pipeline::RenderPipeline::invalid(self.clone(), desc.label.to_string()),
4197                Some(e),
4198            ),
4199        };
4200        #[cfg(feature = "trace")]
4201        if let Some(ref mut trace) = *self.trace.lock() {
4202            use crate::device::trace::IntoTrace;
4203            trace.add(trace::Action::CreateGeneralRenderPipeline {
4204                id: render_pipeline.to_trace(),
4205                desc: desc.to_trace(),
4206            });
4207        }
4208        (render_pipeline, error)
4209    }
4210
4211    pub fn create_render_pipeline_inner(
4212        self: &Arc<Self>,
4213        desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4214    ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreateRenderPipelineError> {
4215        use wgt::TextureFormatFeatureFlags as Tfff;
4216
4217        self.check_is_valid()?;
4218
4219        let mut shader_binding_sizes = FastHashMap::default();
4220
4221        let color_targets = desc
4222            .fragment
4223            .as_ref()
4224            .map_or(&[][..], |fragment| &fragment.targets);
4225        let depth_stencil_state = desc.depth_stencil.as_ref();
4226
4227        check_color_attachment_count(color_targets.len(), self.limits.max_color_attachments)?;
4228
4229        {
4230            let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
4231                color_targets.iter().filter_map(|x| x.as_ref()).collect();
4232            if !cts.is_empty() && {
4233                let first = &cts[0];
4234                cts[1..]
4235                    .iter()
4236                    .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
4237            } {
4238                self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
4239            }
4240        }
4241
4242        let mut io = validation::StageIo::default();
4243        let mut validated_stages = wgt::ShaderStages::empty();
4244
4245        let mut vertex_steps;
4246        let mut hal_vertex_buffer_layouts;
4247        let mut total_attributes;
4248        let mut dual_source_blending = false;
4249        let mut has_depth_attachment = false;
4250        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4251            if vertex.buffers.len() > self.limits.max_vertex_buffers as usize {
4252                return Err(pipeline::CreateRenderPipelineError::TooManyVertexBuffers {
4253                    given: vertex.buffers.len() as u32,
4254                    limit: self.limits.max_vertex_buffers,
4255                });
4256            }
4257
4258            vertex_steps = Vec::with_capacity(vertex.buffers.len());
4259            hal_vertex_buffer_layouts = Vec::with_capacity(vertex.buffers.len());
4260            total_attributes = 0;
4261            for (i, vb_state) in vertex.buffers.iter().enumerate() {
4262                let Some(vb_state) = vb_state else {
4263                    vertex_steps.push(None);
4264                    hal_vertex_buffer_layouts.push(None);
4265                    continue;
4266                };
4267
4268                // https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpuvertexbufferlayout
4269
4270                if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
4271                    return Err(pipeline::CreateRenderPipelineError::VertexStrideTooLarge {
4272                        index: i as u32,
4273                        given: vb_state.array_stride as u32,
4274                        limit: self.limits.max_vertex_buffer_array_stride,
4275                    });
4276                }
4277                if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
4278                    return Err(pipeline::CreateRenderPipelineError::UnalignedVertexStride {
4279                        index: i as u32,
4280                        stride: vb_state.array_stride,
4281                    });
4282                }
4283
4284                let max_stride = if vb_state.array_stride == 0 {
4285                    self.limits.max_vertex_buffer_array_stride as u64
4286                } else {
4287                    vb_state.array_stride
4288                };
4289                let mut last_stride = 0;
4290                for attribute in vb_state.attributes.iter() {
4291                    let attribute_stride = attribute.offset + attribute.format.size();
4292                    if attribute_stride > max_stride {
4293                        return Err(
4294                            pipeline::CreateRenderPipelineError::VertexAttributeStrideTooLarge {
4295                                location: attribute.shader_location,
4296                                given: attribute_stride as u32,
4297                                limit: max_stride as u32,
4298                            },
4299                        );
4300                    }
4301
4302                    let required_offset_alignment = attribute.format.size().min(4);
4303                    if attribute.offset % required_offset_alignment != 0 {
4304                        return Err(
4305                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4306                                location: attribute.shader_location,
4307                                offset: attribute.offset,
4308                            },
4309                        );
4310                    }
4311
4312                    if attribute.shader_location >= self.limits.max_vertex_attributes {
4313                        return Err(
4314                            pipeline::CreateRenderPipelineError::VertexAttributeLocationTooLarge {
4315                                given: attribute.shader_location,
4316                                limit: self.limits.max_vertex_attributes,
4317                            },
4318                        );
4319                    }
4320
4321                    last_stride = last_stride.max(attribute_stride);
4322                }
4323
4324                vertex_steps.push(Some(pipeline::VertexStep {
4325                    stride: vb_state.array_stride,
4326                    last_stride,
4327                    mode: vb_state.step_mode,
4328                }));
4329                hal_vertex_buffer_layouts.push(if vb_state.attributes.is_empty() {
4330                    None
4331                } else {
4332                    Some(hal::VertexBufferLayout {
4333                        array_stride: vb_state.array_stride,
4334                        step_mode: vb_state.step_mode,
4335                        attributes: vb_state.attributes.as_ref(),
4336                    })
4337                });
4338
4339                for attribute in vb_state.attributes.iter() {
4340                    if attribute.offset >= 0x10000000 {
4341                        return Err(
4342                            pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4343                                location: attribute.shader_location,
4344                                offset: attribute.offset,
4345                            },
4346                        );
4347                    }
4348
4349                    if let wgt::VertexFormat::Float64
4350                    | wgt::VertexFormat::Float64x2
4351                    | wgt::VertexFormat::Float64x3
4352                    | wgt::VertexFormat::Float64x4 = attribute.format
4353                    {
4354                        self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
4355                    }
4356
4357                    let previous = io.varyings.insert(
4358                        attribute.shader_location,
4359                        validation::InterfaceVar::vertex_attribute(attribute.format),
4360                    );
4361
4362                    if previous.is_some() {
4363                        return Err(pipeline::CreateRenderPipelineError::ShaderLocationClash(
4364                            attribute.shader_location,
4365                        ));
4366                    }
4367                }
4368                total_attributes += vb_state.attributes.len();
4369            }
4370
4371            if total_attributes > self.limits.max_vertex_attributes as usize {
4372                return Err(
4373                    pipeline::CreateRenderPipelineError::TooManyVertexAttributes {
4374                        given: total_attributes as u32,
4375                        limit: self.limits.max_vertex_attributes,
4376                    },
4377                );
4378            }
4379        } else {
4380            vertex_steps = Vec::new();
4381            hal_vertex_buffer_layouts = Vec::new();
4382        };
4383
4384        if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
4385            return Err(
4386                pipeline::CreateRenderPipelineError::StripIndexFormatForNonStripTopology {
4387                    strip_index_format: desc.primitive.strip_index_format,
4388                    topology: desc.primitive.topology,
4389                },
4390            );
4391        }
4392
4393        if desc.primitive.unclipped_depth {
4394            self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
4395        }
4396
4397        if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
4398            self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
4399        }
4400        if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
4401            self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
4402        }
4403
4404        if desc.primitive.conservative {
4405            self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
4406        }
4407
4408        if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
4409            return Err(
4410                pipeline::CreateRenderPipelineError::ConservativeRasterizationNonFillPolygonMode,
4411            );
4412        }
4413
4414        let mut target_specified = false;
4415
4416        for (i, cs) in color_targets.iter().enumerate() {
4417            if let Some(cs) = cs.as_ref() {
4418                target_specified = true;
4419                let error = 'error: {
4420                    // This is expected to be the operative check for illegal write mask
4421                    // values (larger than 15), because WebGPU requires that it be validated
4422                    // on the device timeline.
4423                    if cs.write_mask.contains_unknown_bits() {
4424                        break 'error Some(ColorStateError::InvalidWriteMask(cs.write_mask));
4425                    }
4426
4427                    let format_features = self.describe_format_features(cs.format)?;
4428                    if !format_features
4429                        .allowed_usages
4430                        .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4431                    {
4432                        break 'error Some(ColorStateError::FormatNotRenderable(cs.format));
4433                    }
4434                    if cs.blend.is_some() && !format_features.flags.contains(Tfff::BLENDABLE) {
4435                        break 'error Some(ColorStateError::FormatNotBlendable(cs.format));
4436                    }
4437                    if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
4438                        break 'error Some(ColorStateError::FormatNotColor(cs.format));
4439                    }
4440
4441                    if desc.multisample.count > 1
4442                        && !format_features
4443                            .flags
4444                            .sample_count_supported(desc.multisample.count)
4445                    {
4446                        break 'error Some(ColorStateError::InvalidSampleCount(
4447                            desc.multisample.count,
4448                            cs.format,
4449                            cs.format
4450                                .guaranteed_format_features(self.features)
4451                                .flags
4452                                .supported_sample_counts(),
4453                            self.adapter
4454                                .get_texture_format_features(cs.format)
4455                                .flags
4456                                .supported_sample_counts(),
4457                        ));
4458                    }
4459
4460                    if let Some(blend_mode) = cs.blend {
4461                        for component in [&blend_mode.color, &blend_mode.alpha] {
4462                            for factor in [component.src_factor, component.dst_factor] {
4463                                if factor.ref_second_blend_source() {
4464                                    self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
4465                                    if i == 0 {
4466                                        dual_source_blending = true;
4467                                    } else {
4468                                        break 'error Some(
4469                                            ColorStateError::BlendFactorOnUnsupportedTarget {
4470                                                factor,
4471                                                target: i as u32,
4472                                            },
4473                                        );
4474                                    }
4475                                }
4476
4477                                if [wgt::BlendOperation::Min, wgt::BlendOperation::Max]
4478                                    .contains(&component.operation)
4479                                    && factor != wgt::BlendFactor::One
4480                                {
4481                                    break 'error Some(ColorStateError::InvalidMinMaxBlendFactor {
4482                                        factor,
4483                                        target: i as u32,
4484                                    });
4485                                }
4486                            }
4487                        }
4488                    }
4489
4490                    break 'error None;
4491                };
4492                if let Some(e) = error {
4493                    return Err(pipeline::CreateRenderPipelineError::ColorState(i as u8, e));
4494                }
4495            }
4496        }
4497
4498        if dual_source_blending && color_targets.len() > 1 {
4499            return Err(
4500                pipeline::CreateRenderPipelineError::DualSourceBlendingWithMultipleColorTargets {
4501                    count: color_targets.len(),
4502                },
4503            );
4504        }
4505
4506        validation::validate_color_attachment_bytes_per_sample(
4507            color_targets.iter().flatten().map(|cs| cs.format),
4508            self.limits.max_color_attachment_bytes_per_sample,
4509        )
4510        .map_err(pipeline::CreateRenderPipelineError::ColorAttachment)?;
4511
4512        if let Some(ds) = depth_stencil_state {
4513            // See <https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-gpudepthstencilstate>.
4514            target_specified = true;
4515            let error = 'error: {
4516                if !ds.format.is_depth_stencil_format() {
4517                    // This error case is not redundant with the aspect check below when
4518                    // neither depth nor stencil is enabled at all.
4519                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepthOrStencil(
4520                        ds.format,
4521                    ));
4522                }
4523
4524                let format_features = self.describe_format_features(ds.format)?;
4525                if !format_features
4526                    .allowed_usages
4527                    .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4528                {
4529                    break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
4530                        ds.format,
4531                    ));
4532                }
4533
4534                let aspect = hal::FormatAspects::from(ds.format);
4535                if aspect.contains(hal::FormatAspects::DEPTH) {
4536                    has_depth_attachment = true;
4537                } else if ds.is_depth_enabled() {
4538                    break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
4539                }
4540                if has_depth_attachment {
4541                    let Some(depth_write_enabled) = ds.depth_write_enabled else {
4542                        break 'error Some(
4543                            pipeline::DepthStencilStateError::MissingDepthWriteEnabled(ds.format),
4544                        );
4545                    };
4546
4547                    let depth_compare_required = depth_write_enabled
4548                        || ds.stencil.front.depth_fail_op != wgt::StencilOperation::Keep
4549                        || ds.stencil.back.depth_fail_op != wgt::StencilOperation::Keep;
4550                    if depth_compare_required && ds.depth_compare.is_none() {
4551                        break 'error Some(pipeline::DepthStencilStateError::MissingDepthCompare(
4552                            ds.format,
4553                        ));
4554                    }
4555                }
4556
4557                if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
4558                    break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
4559                        ds.format,
4560                    ));
4561                }
4562                if desc.multisample.count > 1
4563                    && !format_features
4564                        .flags
4565                        .sample_count_supported(desc.multisample.count)
4566                {
4567                    break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
4568                        desc.multisample.count,
4569                        ds.format,
4570                        ds.format
4571                            .guaranteed_format_features(self.features)
4572                            .flags
4573                            .supported_sample_counts(),
4574                        self.adapter
4575                            .get_texture_format_features(ds.format)
4576                            .flags
4577                            .supported_sample_counts(),
4578                    ));
4579                }
4580
4581                break 'error None;
4582            };
4583            if let Some(e) = error {
4584                return Err(pipeline::CreateRenderPipelineError::DepthStencilState(e));
4585            }
4586
4587            if ds.bias.clamp != 0.0 {
4588                self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
4589            }
4590
4591            if (ds.bias.is_enabled() || ds.bias.clamp != 0.0)
4592                && !desc.primitive.topology.is_triangles()
4593            {
4594                return Err(pipeline::CreateRenderPipelineError::DepthStencilState(
4595                    pipeline::DepthStencilStateError::DepthBiasWithIncompatibleTopology(
4596                        desc.primitive.topology,
4597                    ),
4598                ));
4599            }
4600        }
4601
4602        if !target_specified {
4603            return Err(pipeline::CreateRenderPipelineError::NoTargetSpecified);
4604        }
4605
4606        let is_auto_layout = desc.layout.is_none();
4607
4608        // Get the pipeline layout from the desc if it is provided.
4609        let pipeline_layout = match desc.layout {
4610            Some(pipeline_layout) => {
4611                pipeline_layout.same_device(self)?;
4612                pipeline_layout.check_valid()?;
4613                Some(pipeline_layout)
4614            }
4615            None => None,
4616        };
4617
4618        let mut binding_layout_source = match pipeline_layout {
4619            Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4620            None => validation::BindingLayoutSource::new_derived(&self.limits),
4621        };
4622
4623        let samples = {
4624            let sc = desc.multisample.count;
4625            if sc == 0 || sc > 32 || !sc.is_power_of_two() {
4626                return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc));
4627            }
4628            sc
4629        };
4630
4631        let mut vertex_stage = None;
4632        let mut task_stage = None;
4633        let mut mesh_stage = None;
4634        let mut _vertex_entry_point_name = String::new();
4635        let mut _task_entry_point_name = String::new();
4636        let mut _mesh_entry_point_name = String::new();
4637        let mut immediate_slots_required = naga::valid::ImmediateSlots::default();
4638        match desc.vertex {
4639            pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
4640                vertex_stage = {
4641                    let stage_desc = &vertex.stage;
4642                    let stage = validation::ShaderStageForValidation::Vertex {
4643                        topology: desc.primitive.topology,
4644                        compare_function: desc.depth_stencil.as_ref().and_then(|d| d.depth_compare),
4645                    };
4646                    let stage_bit = stage.to_wgt_bit();
4647
4648                    let vertex_shader_module = &stage_desc.module;
4649                    vertex_shader_module.same_device(self)?;
4650
4651                    let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4652                        stage: stage_bit,
4653                        error,
4654                    };
4655
4656                    _vertex_entry_point_name = vertex_shader_module
4657                        .finalize_entry_point_name(
4658                            stage.to_naga(),
4659                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4660                        )
4661                        .map_err(stage_err)?;
4662
4663                    if let Some(interface) = vertex_shader_module.interface.interface() {
4664                        immediate_slots_required |= interface
4665                            .immediate_slots_required(stage.to_naga(), &_vertex_entry_point_name);
4666                        io = interface
4667                            .check_stage(
4668                                &mut binding_layout_source,
4669                                &mut shader_binding_sizes,
4670                                &_vertex_entry_point_name,
4671                                stage,
4672                                io,
4673                                Some(desc.primitive.topology),
4674                            )
4675                            .map_err(stage_err)?;
4676                        validated_stages |= stage_bit;
4677                    }
4678                    Some(hal::ProgrammableStage {
4679                        module: vertex_shader_module.raw(),
4680                        entry_point: &_vertex_entry_point_name,
4681                        constants: &stage_desc.constants,
4682                        zero_initialize_workgroup_memory: stage_desc
4683                            .zero_initialize_workgroup_memory,
4684                    })
4685                };
4686            }
4687            pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
4688                self.require_features(wgt::Features::EXPERIMENTAL_MESH_SHADER)?;
4689
4690                task_stage = if let Some(task) = task {
4691                    let stage_desc = &task.stage;
4692                    let stage = validation::ShaderStageForValidation::Task;
4693                    let stage_bit = stage.to_wgt_bit();
4694                    let task_shader_module = &stage_desc.module;
4695                    task_shader_module.same_device(self)?;
4696
4697                    let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4698                        stage: stage_bit,
4699                        error,
4700                    };
4701
4702                    _task_entry_point_name = task_shader_module
4703                        .finalize_entry_point_name(
4704                            stage.to_naga(),
4705                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4706                        )
4707                        .map_err(stage_err)?;
4708
4709                    if let Some(interface) = task_shader_module.interface.interface() {
4710                        immediate_slots_required |= interface
4711                            .immediate_slots_required(stage.to_naga(), &_task_entry_point_name);
4712                        io = interface
4713                            .check_stage(
4714                                &mut binding_layout_source,
4715                                &mut shader_binding_sizes,
4716                                &_task_entry_point_name,
4717                                stage,
4718                                io,
4719                                Some(desc.primitive.topology),
4720                            )
4721                            .map_err(stage_err)?;
4722                        validated_stages |= stage_bit;
4723                    }
4724                    Some(hal::ProgrammableStage {
4725                        module: task_shader_module.raw(),
4726                        entry_point: &_task_entry_point_name,
4727                        constants: &stage_desc.constants,
4728                        zero_initialize_workgroup_memory: stage_desc
4729                            .zero_initialize_workgroup_memory,
4730                    })
4731                } else {
4732                    None
4733                };
4734                mesh_stage = {
4735                    let stage_desc = &mesh.stage;
4736                    let stage = validation::ShaderStageForValidation::Mesh;
4737                    let stage_bit = stage.to_wgt_bit();
4738                    let mesh_shader_module = &stage_desc.module;
4739                    mesh_shader_module.same_device(self)?;
4740
4741                    let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4742                        stage: stage_bit,
4743                        error,
4744                    };
4745
4746                    _mesh_entry_point_name = mesh_shader_module
4747                        .finalize_entry_point_name(
4748                            stage.to_naga(),
4749                            stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4750                        )
4751                        .map_err(stage_err)?;
4752
4753                    if let Some(interface) = mesh_shader_module.interface.interface() {
4754                        immediate_slots_required |= interface
4755                            .immediate_slots_required(stage.to_naga(), &_mesh_entry_point_name);
4756                        io = interface
4757                            .check_stage(
4758                                &mut binding_layout_source,
4759                                &mut shader_binding_sizes,
4760                                &_mesh_entry_point_name,
4761                                stage,
4762                                io,
4763                                Some(desc.primitive.topology),
4764                            )
4765                            .map_err(stage_err)?;
4766                        validated_stages |= stage_bit;
4767                    }
4768                    Some(hal::ProgrammableStage {
4769                        module: mesh_shader_module.raw(),
4770                        entry_point: &_mesh_entry_point_name,
4771                        constants: &stage_desc.constants,
4772                        zero_initialize_workgroup_memory: stage_desc
4773                            .zero_initialize_workgroup_memory,
4774                    })
4775                };
4776            }
4777        }
4778
4779        let fragment_entry_point_name;
4780        let fragment_stage = match desc.fragment {
4781            Some(ref fragment_state) => {
4782                let stage = validation::ShaderStageForValidation::Fragment {
4783                    dual_source_blending,
4784                    has_depth_attachment,
4785                };
4786                let stage_bit = stage.to_wgt_bit();
4787
4788                let shader_module = &fragment_state.stage.module;
4789                shader_module.same_device(self)?;
4790
4791                let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4792                    stage: stage_bit,
4793                    error,
4794                };
4795
4796                fragment_entry_point_name = shader_module
4797                    .finalize_entry_point_name(
4798                        stage.to_naga(),
4799                        fragment_state
4800                            .stage
4801                            .entry_point
4802                            .as_ref()
4803                            .map(|ep| ep.as_ref()),
4804                    )
4805                    .map_err(stage_err)?;
4806
4807                if let Some(interface) = shader_module.interface.interface() {
4808                    immediate_slots_required |= interface
4809                        .immediate_slots_required(stage.to_naga(), &fragment_entry_point_name);
4810                    io = interface
4811                        .check_stage(
4812                            &mut binding_layout_source,
4813                            &mut shader_binding_sizes,
4814                            &fragment_entry_point_name,
4815                            stage,
4816                            io,
4817                            Some(desc.primitive.topology),
4818                        )
4819                        .map_err(stage_err)?;
4820                    validated_stages |= stage_bit;
4821                }
4822
4823                Some(hal::ProgrammableStage {
4824                    module: shader_module.raw(),
4825                    entry_point: &fragment_entry_point_name,
4826                    constants: &fragment_state.stage.constants,
4827                    zero_initialize_workgroup_memory: fragment_state
4828                        .stage
4829                        .zero_initialize_workgroup_memory,
4830                })
4831            }
4832            None => None,
4833        };
4834
4835        if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
4836            for (i, output) in io.varyings.iter() {
4837                match color_targets.get(*i as usize) {
4838                    Some(Some(state)) => {
4839                        validation::check_texture_format(state.format, &output.ty).map_err(
4840                            |pipeline| {
4841                                pipeline::CreateRenderPipelineError::ColorState(
4842                                    *i as u8,
4843                                    ColorStateError::IncompatibleFormat {
4844                                        pipeline,
4845                                        shader: output.ty,
4846                                    },
4847                                )
4848                            },
4849                        )?;
4850                    }
4851                    _ => {
4852                        log::debug!(
4853                            "The fragment stage {:?} output @location({}) values are ignored",
4854                            fragment_stage
4855                                .as_ref()
4856                                .map_or("", |stage| stage.entry_point),
4857                            i
4858                        );
4859                    }
4860                }
4861            }
4862        }
4863        let last_stage = match desc.fragment {
4864            Some(_) => wgt::ShaderStages::FRAGMENT,
4865            None => wgt::ShaderStages::VERTEX,
4866        };
4867        if is_auto_layout && !validated_stages.contains(last_stage) {
4868            return Err(pipeline::ImplicitLayoutError::ReflectionError(last_stage).into());
4869        }
4870
4871        let pipeline_layout = match binding_layout_source {
4872            validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4873            validation::BindingLayoutSource::Derived(entries) => {
4874                let immediate_size = {
4875                    let immediate_size_of = |sm: &pipeline::ShaderModule| {
4876                        sm.interface.interface().map(|i| i.immediate_size)
4877                    };
4878                    let vertex = match desc.vertex {
4879                        pipeline::RenderPipelineVertexProcessor::Vertex(ref v) => {
4880                            immediate_size_of(&v.stage.module)
4881                        }
4882                        pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => task
4883                            .as_ref()
4884                            .and_then(|t| immediate_size_of(&t.stage.module))
4885                            .max(immediate_size_of(&mesh.stage.module)),
4886                    };
4887                    let fragment = desc
4888                        .fragment
4889                        .as_ref()
4890                        .and_then(|f| immediate_size_of(&f.stage.module));
4891                    vertex.max(fragment).unwrap_or(0)
4892                };
4893                self.create_derived_pipeline_layout(entries, immediate_size)?
4894            }
4895        };
4896
4897        if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4898            let bind_groups_plus_vertex_buffers =
4899                u32::try_from(pipeline_layout.bind_group_layouts.len() + vertex.buffers.len())
4900                    .unwrap();
4901            if bind_groups_plus_vertex_buffers > self.limits.max_bind_groups_plus_vertex_buffers {
4902                return Err(
4903                    pipeline::CreateRenderPipelineError::TooManyBindGroupsPlusVertexBuffers {
4904                        given: bind_groups_plus_vertex_buffers,
4905                        limit: self.limits.max_bind_groups_plus_vertex_buffers,
4906                    },
4907                );
4908            }
4909
4910            let given = pipeline_layout
4911                .buffers_and_acceleration_structures_in_vertex_stage
4912                .saturating_add(vertex.buffers.len() as u32);
4913            if !self
4914                .instance_flags
4915                .contains(wgt::InstanceFlags::STRICT_WEBGPU_COMPLIANCE)
4916            {
4917                let limit = self
4918                    .limits
4919                    .max_buffers_and_acceleration_structures_per_shader_stage;
4920                if given > limit {
4921                    return Err(
4922                    pipeline::CreateRenderPipelineError::TooManyBuffersAndAccelerationStructuresInVertexStage {
4923                        given,
4924                        limit,
4925                    },
4926                );
4927                }
4928            }
4929        }
4930
4931        // Multiview is only supported if the feature is enabled
4932        if let Some(mv_mask) = desc.multiview_mask {
4933            self.require_features(wgt::Features::MULTIVIEW)?;
4934            if !(mv_mask.get() + 1).is_power_of_two() {
4935                self.require_features(wgt::Features::SELECTIVE_MULTIVIEW)?;
4936            }
4937        }
4938
4939        if !self
4940            .downlevel
4941            .flags
4942            .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
4943        {
4944            for (binding, size) in shader_binding_sizes.iter() {
4945                if size.get() % 16 != 0 {
4946                    return Err(pipeline::CreateRenderPipelineError::UnalignedShader {
4947                        binding: binding.binding,
4948                        group: binding.group,
4949                        size: size.get(),
4950                    });
4951                }
4952            }
4953        }
4954
4955        let late_sized_buffer_groups =
4956            Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4957
4958        let cache = match desc.cache {
4959            Some(cache) => {
4960                cache.same_device(self)?;
4961                Some(cache)
4962            }
4963            None => None,
4964        };
4965
4966        let is_mesh = mesh_stage.is_some();
4967        let has_task_shader = task_stage.is_some();
4968        let raw = {
4969            let pipeline_desc = hal::RenderPipelineDescriptor {
4970                label: desc.label.to_hal(self.instance_flags),
4971                layout: pipeline_layout.raw()?,
4972                vertex_processor: match vertex_stage {
4973                    Some(vertex_stage) => hal::VertexProcessor::Standard {
4974                        vertex_buffers: &hal_vertex_buffer_layouts,
4975                        vertex_stage,
4976                    },
4977                    None => hal::VertexProcessor::Mesh {
4978                        task_stage,
4979                        mesh_stage: mesh_stage.unwrap(),
4980                    },
4981                },
4982                primitive: desc.primitive,
4983                depth_stencil: desc.depth_stencil.clone(),
4984                multisample: desc.multisample,
4985                fragment_stage,
4986                color_targets,
4987                multiview_mask: desc.multiview_mask,
4988                cache: cache.as_ref().map(|it| it.raw()),
4989            };
4990            unsafe { self.raw().create_render_pipeline(&pipeline_desc) }.map_err(
4991                |err| match err {
4992                    hal::PipelineError::Device(error) => {
4993                        pipeline::CreateRenderPipelineError::Device(self.handle_hal_error(error))
4994                    }
4995                    hal::PipelineError::Linkage(stage, msg) => {
4996                        pipeline::CreateRenderPipelineError::Internal { stage, error: msg }
4997                    }
4998                    hal::PipelineError::EntryPoint(stage) => {
4999                        pipeline::CreateRenderPipelineError::Internal {
5000                            stage: hal::auxil::map_naga_stage(stage),
5001                            error: ENTRYPOINT_FAILURE_ERROR.to_string(),
5002                        }
5003                    }
5004                    hal::PipelineError::PipelineConstants(stage, error) => {
5005                        pipeline::CreateRenderPipelineError::PipelineConstants { stage, error }
5006                    }
5007                },
5008            )?
5009        };
5010
5011        let pass_context = RenderPassContext {
5012            attachments: AttachmentData {
5013                colors: color_targets
5014                    .iter()
5015                    .map(|state| state.as_ref().map(|s| s.format))
5016                    .collect(),
5017                resolves: ArrayVec::new(),
5018                depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
5019            },
5020            sample_count: samples,
5021            multiview_mask: desc.multiview_mask,
5022        };
5023
5024        let mut flags = pipeline::PipelineFlags::empty();
5025        for state in color_targets.iter().filter_map(|s| s.as_ref()) {
5026            if let Some(ref bs) = state.blend {
5027                if bs.color.uses_constant() | bs.alpha.uses_constant() {
5028                    flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
5029                }
5030            }
5031        }
5032        if let Some(ds) = depth_stencil_state.as_ref() {
5033            if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
5034                flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
5035            }
5036            if !ds.is_depth_read_only() {
5037                flags |= pipeline::PipelineFlags::WRITES_DEPTH;
5038            }
5039            if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
5040                flags |= pipeline::PipelineFlags::WRITES_STENCIL;
5041            }
5042        }
5043        let shader_modules = {
5044            let mut shader_modules = ArrayVec::new();
5045            match desc.vertex {
5046                pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
5047                    shader_modules.push(vertex.stage.module)
5048                }
5049                pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
5050                    if let Some(task) = task {
5051                        shader_modules.push(task.stage.module);
5052                    }
5053                    shader_modules.push(mesh.stage.module);
5054                }
5055            }
5056            shader_modules.extend(desc.fragment.map(|f| f.stage.module));
5057            shader_modules
5058        };
5059
5060        let pipeline = pipeline::RenderPipeline {
5061            state: ResourceState::Valid(pipeline::RenderPipelineState {
5062                raw: ManuallyDrop::new(raw),
5063                layout: pipeline_layout.clone(),
5064            }),
5065            device: self.clone(),
5066            pass_context,
5067            _shader_modules: shader_modules,
5068            flags,
5069            topology: desc.primitive.topology,
5070            strip_index_format: desc.primitive.strip_index_format,
5071            vertex_steps,
5072            late_sized_buffer_groups,
5073            immediate_slots_required,
5074            label: desc.label.to_string(),
5075            tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
5076            is_mesh,
5077            has_task_shader,
5078        };
5079
5080        let pipeline = Arc::new(pipeline);
5081
5082        if is_auto_layout {
5083            for bgl in pipeline_layout.bind_group_layouts.iter() {
5084                let Some(bgl) = bgl else {
5085                    continue;
5086                };
5087
5088                // `bind_group_layouts` might contain duplicate entries, so we need to ignore the
5089                // result.
5090                let _ = bgl.exclusive_pipeline.set((&pipeline).into());
5091            }
5092        }
5093
5094        Ok(pipeline)
5095    }
5096
5097    /// # Safety
5098    /// The `data` field on `desc` must have previously been returned from
5099    /// [`crate::global::Global::pipeline_cache_get_data`]
5100    pub unsafe fn create_pipeline_cache(
5101        self: &Arc<Self>,
5102        desc: &pipeline::PipelineCacheDescriptor,
5103    ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
5104        use crate::pipeline_cache;
5105
5106        self.check_is_valid()?;
5107
5108        self.require_features(wgt::Features::PIPELINE_CACHE)?;
5109        let data = if let Some((data, validation_key)) = desc
5110            .data
5111            .as_ref()
5112            .zip(self.raw().pipeline_cache_validation_key())
5113        {
5114            let data = pipeline_cache::validate_pipeline_cache(
5115                data,
5116                &self.adapter.raw.info,
5117                validation_key,
5118            );
5119            match data {
5120                Ok(data) => Some(data),
5121                Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
5122                // If the error was unavoidable and we are asked to fallback, do so
5123                Err(_) => None,
5124            }
5125        } else {
5126            None
5127        };
5128        let cache_desc = hal::PipelineCacheDescriptor {
5129            data,
5130            label: desc.label.to_hal(self.instance_flags),
5131        };
5132        let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
5133            Ok(raw) => raw,
5134            Err(e) => match e {
5135                hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
5136            },
5137        };
5138        let cache = pipeline::PipelineCache {
5139            device: self.clone(),
5140            label: desc.label.to_string(),
5141            // This would be none in the error condition, which we don't implement yet
5142            raw: ManuallyDrop::new(raw),
5143        };
5144
5145        let cache = Arc::new(cache);
5146
5147        Ok(cache)
5148    }
5149
5150    fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
5151        // Variant of adapter.get_texture_format_features that takes device features into account
5152        use wgt::TextureFormatFeatureFlags as tfsc;
5153        let mut format_features = self.adapter.get_texture_format_features(format);
5154        if (format == TextureFormat::R32Float
5155            || format == TextureFormat::Rg32Float
5156            || format == TextureFormat::Rgba32Float)
5157            && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
5158        {
5159            format_features.flags.set(tfsc::FILTERABLE, false);
5160        }
5161        format_features
5162    }
5163
5164    pub(crate) fn describe_format_features(
5165        &self,
5166        format: TextureFormat,
5167    ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
5168        self.require_features(format.required_features())?;
5169
5170        let using_device_features = self
5171            .features
5172            .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
5173        // If we're running downlevel, we need to manually ask the backend what
5174        // we can use as we can't trust WebGPU.
5175        let downlevel = !self
5176            .downlevel
5177            .flags
5178            .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
5179
5180        if using_device_features || downlevel {
5181            Ok(self.get_texture_format_features(format))
5182        } else {
5183            Ok(format.guaranteed_format_features(self.features))
5184        }
5185    }
5186
5187    #[cfg(feature = "replay")]
5188    pub(crate) fn wait_for_submit(
5189        &self,
5190        submission_index: crate::SubmissionIndex,
5191    ) -> Result<(), DeviceError> {
5192        let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }
5193            .map_err(|e| self.handle_hal_error(e))?;
5194        if last_done_index < submission_index {
5195            unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) }
5196                .map_err(|e| self.handle_hal_error(e))?;
5197            if let Some(queue) = self.get_queue() {
5198                let closures = queue.lock_life().triage_submissions(submission_index);
5199                assert!(
5200                    closures.is_empty(),
5201                    "wait_for_submit is not expected to work with closures"
5202                );
5203            }
5204        }
5205        Ok(())
5206    }
5207
5208    pub fn create_query_set(
5209        self: &Arc<Self>,
5210        desc: &resource::QuerySetDescriptor,
5211    ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
5212        use resource::CreateQuerySetError as Error;
5213
5214        self.check_is_valid()?;
5215
5216        match desc.ty {
5217            wgt::QueryType::Occlusion => {}
5218            wgt::QueryType::Timestamp => {
5219                self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
5220            }
5221            wgt::QueryType::PipelineStatistics(..) => {
5222                self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
5223            }
5224        }
5225
5226        if desc.count == 0 {
5227            return Err(Error::ZeroCount);
5228        }
5229
5230        if desc.count > wgt::QUERY_SET_MAX_QUERIES {
5231            return Err(Error::TooManyQueries {
5232                count: desc.count,
5233                maximum: wgt::QUERY_SET_MAX_QUERIES,
5234            });
5235        }
5236
5237        let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
5238
5239        let raw = unsafe { self.raw().create_query_set(&hal_desc) }
5240            .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
5241
5242        let query_set = QuerySet {
5243            raw: Snatchable::new(raw),
5244            device: self.clone(),
5245            label: desc.label.to_string(),
5246            tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
5247            desc: desc.map_label(|_| ()),
5248            initialized_slots: Mutex::new(
5249                rank::QUERY_SET_INITIALIZED_SLOTS,
5250                bit_vec::BitVec::from_elem(desc.count as usize, false),
5251            ),
5252        };
5253
5254        let query_set = Arc::new(query_set);
5255
5256        Ok(query_set)
5257    }
5258
5259    pub fn configure_surface(
5260        self: &Arc<Self>,
5261        surface: &crate::instance::Surface,
5262        config: &wgt::SurfaceConfiguration<Vec<TextureFormat>>,
5263    ) -> Option<present::ConfigureSurfaceError> {
5264        use present::ConfigureSurfaceError as E;
5265        profiling::scope!("surface_configure");
5266
5267        log::debug!("configuring surface with {config:?}");
5268
5269        let error = 'error: {
5270            // User callbacks must not be called while we are holding locks.
5271            let user_callbacks;
5272            {
5273                if let Err(e) = self.check_is_valid() {
5274                    break 'error e.into();
5275                }
5276
5277                let caps = match surface.get_capabilities(&self.adapter) {
5278                    Ok(caps) => caps,
5279                    Err(_) => break 'error E::UnsupportedQueueFamily,
5280                };
5281
5282                let mut hal_view_formats = Vec::new();
5283                for format in config.view_formats.iter() {
5284                    if *format == config.format {
5285                        continue;
5286                    }
5287                    if !caps.formats.iter().any(|fc| fc.format == config.format) {
5288                        break 'error E::UnsupportedFormat {
5289                            requested: config.format,
5290                            available: caps.texture_formats().collect(),
5291                        };
5292                    }
5293                    if config.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
5294                        break 'error E::InvalidViewFormat(*format, config.format);
5295                    }
5296                    hal_view_formats.push(*format);
5297                }
5298
5299                if !hal_view_formats.is_empty() {
5300                    if let Err(missing_flag) =
5301                        self.require_downlevel_flags(wgt::DownlevelFlags::SURFACE_VIEW_FORMATS)
5302                    {
5303                        break 'error E::MissingDownlevelFlags(missing_flag);
5304                    }
5305                }
5306
5307                let maximum_frame_latency = config.desired_maximum_frame_latency.clamp(
5308                    *caps.maximum_frame_latency.start(),
5309                    *caps.maximum_frame_latency.end(),
5310                );
5311                let mut hal_config = hal::SurfaceConfiguration {
5312                    maximum_frame_latency,
5313                    present_mode: config.present_mode,
5314                    composite_alpha_mode: config.alpha_mode,
5315                    format: config.format,
5316                    color_space: config.color_space,
5317                    extent: wgt::Extent3d {
5318                        width: config.width,
5319                        height: config.height,
5320                        depth_or_array_layers: 1,
5321                    },
5322                    usage: conv::map_texture_usage(
5323                        config.usage,
5324                        hal::FormatAspects::COLOR,
5325                        wgt::TextureFormatFeatureFlags::STORAGE_READ_ONLY
5326                            | wgt::TextureFormatFeatureFlags::STORAGE_WRITE_ONLY
5327                            | wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE,
5328                    ),
5329                    view_formats: hal_view_formats,
5330                };
5331
5332                if let Err(error) = validate_surface_configuration(
5333                    &mut hal_config,
5334                    &caps,
5335                    self.limits.max_texture_dimension_2d,
5336                ) {
5337                    break 'error error;
5338                }
5339
5340                // Wait for all work to finish before configuring the surface.
5341                let snatch_guard = self.snatchable_lock.read();
5342
5343                let maintain_result;
5344                (user_callbacks, maintain_result) =
5345                    self.maintain(wgt::PollType::wait_indefinitely(), snatch_guard);
5346
5347                match maintain_result {
5348                    // We're happy
5349                    Ok(wgt::PollStatus::QueueEmpty) => {}
5350                    Ok(wgt::PollStatus::WaitSucceeded) => {
5351                        // After the wait, the queue should be empty. It can only be non-empty
5352                        // if another thread is submitting at the same time.
5353                        break 'error E::GpuWaitTimeout;
5354                    }
5355                    Ok(wgt::PollStatus::Poll) => {
5356                        unreachable!("Cannot get a Poll result from a Wait action.")
5357                    }
5358                    Err(WaitIdleError::Timeout) if cfg!(target_arch = "wasm32") => {
5359                        // On wasm, you cannot actually successfully wait for the surface.
5360                        // However WebGL does not actually require you do this, so ignoring
5361                        // the failure is totally fine. See
5362                        // https://github.com/gfx-rs/wgpu/issues/7363
5363                    }
5364                    Err(e) => {
5365                        break 'error e.into();
5366                    }
5367                }
5368
5369                // All textures must be destroyed before the surface can be re-configured.
5370                if let Some(present) = surface.presentation.lock().take() {
5371                    if present.acquired_texture.is_some() {
5372                        break 'error E::PreviousOutputExists;
5373                    }
5374                }
5375
5376                // TODO: Texture views may still be alive that point to the texture.
5377                // this will allow the user to render to the surface texture, long after
5378                // it has been removed.
5379                //
5380                // https://github.com/gfx-rs/wgpu/issues/4105
5381
5382                let surface_raw = surface.raw(self.backend()).unwrap();
5383                match unsafe { surface_raw.configure(self.raw(), &hal_config) } {
5384                    Ok(()) => (),
5385                    Err(error) => {
5386                        break 'error match error {
5387                            hal::SurfaceError::Outdated
5388                            | hal::SurfaceError::Lost
5389                            | hal::SurfaceError::Occluded
5390                            | hal::SurfaceError::Timeout => E::InvalidSurface,
5391                            hal::SurfaceError::Device(error) => {
5392                                E::Device(self.handle_hal_error(error))
5393                            }
5394                            hal::SurfaceError::Other(message) => {
5395                                log::error!("surface configuration failed: {message}");
5396                                E::InvalidSurface
5397                            }
5398                        }
5399                    }
5400                }
5401
5402                let mut presentation = surface.presentation.lock();
5403                *presentation = Some(present::Presentation {
5404                    device: Arc::clone(self),
5405                    config: config.clone(),
5406                    acquired_texture: None,
5407                });
5408            }
5409
5410            user_callbacks.fire();
5411            return None;
5412        };
5413
5414        Some(error)
5415    }
5416
5417    fn lose(&self, message: &str) {
5418        // Follow the steps at https://gpuweb.github.io/gpuweb/#lose-the-device.
5419
5420        // Mark the device explicitly as invalid. This is checked in various
5421        // places to prevent new work from being submitted.
5422        self.valid.store(false, Ordering::Release);
5423
5424        // 1) Resolve the GPUDevice device.lost promise.
5425        if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
5426            device_lost_closure(DeviceLostReason::Unknown, message.to_string());
5427        }
5428
5429        // 2) Complete any outstanding mapAsync() steps.
5430        // 3) Complete any outstanding onSubmittedWorkDone() steps.
5431
5432        // These parts are passively accomplished by setting valid to false,
5433        // since that will prevent any new work from being added to the queues.
5434        // Future calls to poll_devices will continue to check the work queues
5435        // until they are cleared, and then drop the device.
5436    }
5437
5438    fn release_gpu_resources(&self) {
5439        // This is called when the device is lost, which makes every associated
5440        // resource invalid and unusable. This is an opportunity to release all of
5441        // the underlying gpu resources, even though the objects remain visible to
5442        // the user agent. We purge this memory naturally when resources have been
5443        // moved into the appropriate buckets, so this function just needs to
5444        // initiate movement into those buckets, and it can do that by calling
5445        // "destroy" on all the resources we know about.
5446
5447        // During these iterations, we discard all errors. We don't care!
5448        let trackers = self.trackers.lock();
5449        for buffer in trackers.buffers.used_resources() {
5450            if let Some(buffer) = Weak::upgrade(buffer) {
5451                buffer.destroy();
5452            }
5453        }
5454        for texture in trackers.textures.used_resources() {
5455            if let Some(texture) = Weak::upgrade(texture) {
5456                texture.destroy();
5457            }
5458        }
5459    }
5460
5461    pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
5462        UsageScope::new_pooled(
5463            &self.usage_scopes,
5464            &self.tracker_indices,
5465            self.ordered_buffer_usages,
5466            self.ordered_texture_usages,
5467        )
5468    }
5469
5470    pub fn get_hal_counters(&self) -> wgt::HalCounters {
5471        self.raw().get_internal_counters()
5472    }
5473
5474    pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
5475        self.raw().generate_allocator_report()
5476    }
5477}
5478
5479crate::impl_resource_type!(Device);
5480crate::impl_labeled!(Device);
5481crate::impl_storage_item!(Device);