Skip to main content

wgpu_core/
resource.rs

1use alloc::{borrow::Cow, borrow::ToOwned as _, boxed::Box, string::String, sync::Arc, vec::Vec};
2use core::{
3    borrow::Borrow,
4    fmt,
5    mem::{self, size_of, ManuallyDrop},
6    num::NonZeroU64,
7    ops::Range,
8    ptr::NonNull,
9};
10use smallvec::SmallVec;
11use thiserror::Error;
12use wgt::{
13    error::{ErrorType, WebGpuError},
14    TextureSelector,
15};
16
17#[cfg(feature = "trace")]
18use crate::device::trace;
19use crate::{
20    binding_model::{BindGroup, BindingError},
21    device::{
22        queue, resource::DeferredDestroy, BufferMapPendingClosure, Device, DeviceError,
23        DeviceMismatch, HostMap, MissingDownlevelFlags, MissingFeatures,
24    },
25    hal_label,
26    init_tracker::{BufferInitTracker, TextureInitTracker},
27    lock::{rank, Mutex, RwLock},
28    ray_tracing::{BlasCompactReadyPendingClosure, BlasPrepareCompactError},
29    resource_log,
30    snatch::{SnatchGuard, Snatchable, Snatchable2},
31    timestamp_normalization::TimestampNormalizationBindGroup,
32    track::{SharedTrackerIndexAllocator, TrackerIndex},
33    weak_vec::WeakVec,
34    Label, LabelHelpers, SubmissionIndex,
35};
36
37/// Information about the wgpu-core resource.
38///
39/// Each type representing a `wgpu-core` resource, like [`Device`],
40/// [`Buffer`], etc., contains a `ResourceInfo` which contains
41/// its latest submission index and label.
42///
43/// A resource may need to be retained for any of several reasons:
44/// and any lifetime logic will be handled by `Arc<Resource>` refcount
45///
46/// - The user may hold a reference to it (via a `wgpu::Buffer`, say).
47///
48/// - Other resources may depend on it (a texture view's backing
49///   texture, for example).
50///
51/// - It may be used by commands sent to the GPU that have not yet
52///   finished execution.
53///
54/// [`Device`]: crate::device::resource::Device
55/// [`Buffer`]: crate::resource::Buffer
56#[derive(Debug)]
57pub(crate) struct TrackingData {
58    tracker_index: TrackerIndex,
59    tracker_indices: Arc<SharedTrackerIndexAllocator>,
60}
61
62impl Drop for TrackingData {
63    fn drop(&mut self) {
64        self.tracker_indices.free(self.tracker_index);
65    }
66}
67
68impl TrackingData {
69    pub(crate) fn new(tracker_indices: Arc<SharedTrackerIndexAllocator>) -> Self {
70        Self {
71            tracker_index: tracker_indices.alloc(),
72            tracker_indices,
73        }
74    }
75
76    pub(crate) fn tracker_index(&self) -> TrackerIndex {
77        self.tracker_index
78    }
79}
80
81#[derive(Clone, Debug)]
82#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
83pub struct ResourceErrorIdent {
84    r#type: Cow<'static, str>,
85    label: String,
86}
87
88impl fmt::Display for ResourceErrorIdent {
89    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
90        write!(f, "{} with '{}' label", self.r#type, self.label)
91    }
92}
93
94#[derive(Debug)]
95pub enum ResourceState<T> {
96    Valid(T),
97    Invalid,
98}
99
100impl<T> ResourceState<T> {
101    pub fn as_ref(&self) -> ResourceState<&T> {
102        match self {
103            ResourceState::Valid(v) => ResourceState::Valid(v),
104            ResourceState::Invalid => ResourceState::Invalid,
105        }
106    }
107
108    pub fn valid(self) -> Option<T> {
109        match self {
110            ResourceState::Valid(v) => Some(v),
111            ResourceState::Invalid => None,
112        }
113    }
114}
115
116pub enum DestructibleResourceState<T> {
117    Valid(T),
118    Invalid,
119    Destroyed,
120}
121
122impl<T> DestructibleResourceState<T> {
123    pub fn as_ref(&self) -> DestructibleResourceState<&T> {
124        match self {
125            DestructibleResourceState::Valid(v) => DestructibleResourceState::Valid(v),
126            DestructibleResourceState::Invalid => DestructibleResourceState::Invalid,
127            DestructibleResourceState::Destroyed => DestructibleResourceState::Destroyed,
128        }
129    }
130
131    pub fn take(&mut self) -> DestructibleResourceState<T> {
132        mem::replace(self, DestructibleResourceState::Destroyed)
133    }
134}
135
136#[derive(thiserror::Error, Debug)]
137pub enum InvalidOrDestroyedResourceError {
138    #[error(transparent)]
139    InvalidResource(#[from] InvalidResourceError),
140    #[error(transparent)]
141    DestroyedResource(#[from] DestroyedResourceError),
142}
143
144impl<T> DestructibleResourceState<T> {
145    pub fn maybe_valid(self) -> Option<T> {
146        match self {
147            DestructibleResourceState::Valid(t) => Some(t),
148            DestructibleResourceState::Invalid => None,
149            DestructibleResourceState::Destroyed => None,
150        }
151    }
152}
153
154pub trait ParentDevice: Labeled {
155    fn device(&self) -> &Arc<Device>;
156
157    fn is_equal(self: &Arc<Self>, other: &Arc<Self>) -> bool {
158        Arc::ptr_eq(self, other)
159    }
160
161    fn same_device_as<O: ParentDevice>(&self, other: &O) -> Result<(), DeviceError> {
162        if Arc::ptr_eq(self.device(), other.device()) {
163            Ok(())
164        } else {
165            Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch {
166                res: self.error_ident(),
167                res_device: self.device().error_ident(),
168                target: Some(other.error_ident()),
169                target_device: other.device().error_ident(),
170            })))
171        }
172    }
173
174    fn same_device(&self, device: &Device) -> Result<(), DeviceError> {
175        if core::ptr::eq(&**self.device(), device) {
176            Ok(())
177        } else {
178            Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch {
179                res: self.error_ident(),
180                res_device: self.device().error_ident(),
181                target: None,
182                target_device: device.error_ident(),
183            })))
184        }
185    }
186}
187
188#[macro_export]
189macro_rules! impl_parent_device {
190    ($ty:ident) => {
191        impl $crate::resource::ParentDevice for $ty {
192            fn device(&self) -> &Arc<Device> {
193                &self.device
194            }
195        }
196    };
197}
198
199/// Allow access to the hal resource as guarded by the `SnatchGuard`.
200pub trait RawResourceAccess: ParentDevice {
201    type DynResource: hal::DynResource + ?Sized;
202
203    /// Get access to the raw resource if it is not destroyed.
204    ///
205    /// Returns `None` if the resource has been destroyed. This method
206    /// does not allocate in either case.
207    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource>;
208
209    /// Get access to the raw resource if it is not destroyed.
210    ///
211    /// Returns a full error if the resource has been destroyed. This
212    /// method allocates a label in the error case.
213    fn try_raw<'a>(
214        &'a self,
215        guard: &'a SnatchGuard,
216    ) -> Result<&'a Self::DynResource, DestroyedResourceError> {
217        self.raw(guard)
218            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
219    }
220}
221
222pub trait ResourceType {
223    const TYPE: &'static str;
224}
225
226#[macro_export]
227macro_rules! impl_resource_type {
228    ($ty:ident) => {
229        impl $crate::resource::ResourceType for $ty {
230            const TYPE: &'static str = stringify!($ty);
231        }
232    };
233}
234
235pub trait Labeled: ResourceType {
236    /// Returns a string identifying this resource for logging and errors.
237    ///
238    /// It may be a user-provided string or it may be a placeholder from wgpu.
239    ///
240    /// It is non-empty unless the user-provided string was empty.
241    fn label(&self) -> &str;
242
243    fn error_ident(&self) -> ResourceErrorIdent {
244        ResourceErrorIdent {
245            r#type: Cow::Borrowed(Self::TYPE),
246            label: self.label().to_owned(),
247        }
248    }
249}
250
251#[macro_export]
252macro_rules! impl_labeled {
253    ($ty:ident) => {
254        impl $crate::resource::Labeled for $ty {
255            fn label(&self) -> &str {
256                &self.label
257            }
258        }
259    };
260}
261
262pub(crate) trait Trackable {
263    fn tracker_index(&self) -> TrackerIndex;
264}
265
266#[macro_export]
267macro_rules! impl_trackable {
268    ($ty:ident) => {
269        impl $crate::resource::Trackable for $ty {
270            fn tracker_index(&self) -> $crate::track::TrackerIndex {
271                self.tracking_data.tracker_index()
272            }
273        }
274    };
275}
276
277#[derive(Debug)]
278pub(crate) enum BufferMapState {
279    /// Mapped at creation.
280    Init { staging_buffer: StagingBuffer },
281    /// Waiting for GPU to be done before mapping
282    Waiting(BufferPendingMapping),
283    /// Mapped
284    Active {
285        mapping: hal::BufferMapping,
286        range: hal::MemoryRange,
287        host: HostMap,
288    },
289    /// Not mapped
290    Idle,
291}
292
293#[cfg(send_sync)]
294unsafe impl Send for BufferMapState {}
295#[cfg(send_sync)]
296unsafe impl Sync for BufferMapState {}
297
298#[cfg(send_sync)]
299pub type BufferMapCallback = Box<dyn FnOnce(BufferAccessResult) + Send + 'static>;
300#[cfg(not(send_sync))]
301pub type BufferMapCallback = Box<dyn FnOnce(BufferAccessResult) + 'static>;
302
303pub struct BufferMapOperation {
304    pub host: HostMap,
305    pub callback: Option<BufferMapCallback>,
306}
307
308impl fmt::Debug for BufferMapOperation {
309    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
310        f.debug_struct("BufferMapOperation")
311            .field("host", &self.host)
312            .field("callback", &self.callback.as_ref().map(|_| "?"))
313            .finish()
314    }
315}
316
317#[derive(Clone, Debug, Error)]
318#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
319#[non_exhaustive]
320pub enum BufferAccessError {
321    #[error(transparent)]
322    Device(#[from] DeviceError),
323    #[error("Buffer map failed")]
324    Failed,
325    #[error(transparent)]
326    DestroyedResource(#[from] DestroyedResourceError),
327    #[error("Buffer is already mapped")]
328    AlreadyMapped,
329    #[error("Buffer map is pending")]
330    MapAlreadyPending,
331    #[error(transparent)]
332    MissingBufferUsage(#[from] MissingBufferUsageError),
333    #[error("Buffer is not mapped")]
334    NotMapped,
335    #[error(
336        "Buffer map range must start aligned to `MAP_ALIGNMENT` and end to `COPY_BUFFER_ALIGNMENT`"
337    )]
338    UnalignedRange,
339    #[error("Buffer offset invalid: offset {offset} must be multiple of 8")]
340    UnalignedOffset { offset: wgt::BufferAddress },
341    #[error("Buffer range size invalid: range_size {range_size} must be multiple of 4")]
342    UnalignedRangeSize { range_size: wgt::BufferAddress },
343    #[error("Buffer access out of bounds: index {index} would underrun the buffer (limit: {min})")]
344    OutOfBoundsStartOffsetUnderrun {
345        index: wgt::BufferAddress,
346        min: wgt::BufferAddress,
347    },
348    #[error(
349        "Buffer access out of bounds: start offset {index} would overrun the buffer (limit: {max})"
350    )]
351    OutOfBoundsStartOffsetOverrun {
352        index: wgt::BufferAddress,
353        max: wgt::BufferAddress,
354    },
355    #[error(
356        "Buffer access out of bounds: start offset {index} + size {size} would overrun the buffer (limit: {max})"
357    )]
358    OutOfBoundsEndOffsetOverrun {
359        index: wgt::BufferAddress,
360        size: wgt::BufferAddress,
361        max: wgt::BufferAddress,
362    },
363    #[error("Buffer map aborted")]
364    MapAborted,
365    #[error(transparent)]
366    InvalidResource(#[from] InvalidResourceError),
367    #[error("Map start offset ({offset}) is out-of-bounds for buffer of size {buffer_size}")]
368    MapStartOffsetOverrun {
369        offset: wgt::BufferAddress,
370        buffer_size: wgt::BufferAddress,
371    },
372    #[error(
373        "Map end offset (start at {} + size of {}) is out-of-bounds for buffer of size {}",
374        offset,
375        size,
376        buffer_size
377    )]
378    MapEndOffsetOverrun {
379        offset: wgt::BufferAddress,
380        size: wgt::BufferAddress,
381        buffer_size: wgt::BufferAddress,
382    },
383}
384
385impl WebGpuError for BufferAccessError {
386    fn webgpu_error_type(&self) -> ErrorType {
387        match self {
388            Self::Device(e) => e.webgpu_error_type(),
389            Self::InvalidResource(e) => e.webgpu_error_type(),
390            Self::DestroyedResource(e) => e.webgpu_error_type(),
391
392            Self::Failed
393            | Self::AlreadyMapped
394            | Self::MapAlreadyPending
395            | Self::MissingBufferUsage(_)
396            | Self::NotMapped
397            | Self::UnalignedRange
398            | Self::UnalignedOffset { .. }
399            | Self::UnalignedRangeSize { .. }
400            | Self::OutOfBoundsStartOffsetUnderrun { .. }
401            | Self::OutOfBoundsStartOffsetOverrun { .. }
402            | Self::OutOfBoundsEndOffsetOverrun { .. }
403            | Self::MapAborted
404            | Self::MapStartOffsetOverrun { .. }
405            | Self::MapEndOffsetOverrun { .. } => ErrorType::Validation,
406        }
407    }
408}
409
410#[derive(Clone, Debug, Error)]
411#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
412#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")]
413pub struct MissingBufferUsageError {
414    pub(crate) res: ResourceErrorIdent,
415    pub(crate) actual: wgt::BufferUsages,
416    pub(crate) expected: wgt::BufferUsages,
417}
418
419impl WebGpuError for MissingBufferUsageError {
420    fn webgpu_error_type(&self) -> ErrorType {
421        ErrorType::Validation
422    }
423}
424
425#[derive(Clone, Debug, Error)]
426#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")]
427pub struct MissingTextureUsageError {
428    pub(crate) res: ResourceErrorIdent,
429    pub(crate) actual: wgt::TextureUsages,
430    pub(crate) expected: wgt::TextureUsages,
431}
432
433impl WebGpuError for MissingTextureUsageError {
434    fn webgpu_error_type(&self) -> ErrorType {
435        ErrorType::Validation
436    }
437}
438
439#[derive(Clone, Debug, Error)]
440#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
441#[error("{0} has been destroyed")]
442pub struct DestroyedResourceError(pub ResourceErrorIdent);
443
444impl WebGpuError for DestroyedResourceError {
445    fn webgpu_error_type(&self) -> ErrorType {
446        ErrorType::Validation
447    }
448}
449
450#[derive(Clone, Debug, Error)]
451#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
452#[error("{0} is invalid")]
453pub struct InvalidResourceError(pub ResourceErrorIdent);
454
455impl WebGpuError for InvalidResourceError {
456    fn webgpu_error_type(&self) -> ErrorType {
457        ErrorType::Validation
458    }
459}
460
461pub enum Fallible<T: ParentDevice> {
462    Valid(Arc<T>),
463    Invalid(Arc<String>),
464}
465
466impl<T: ParentDevice> Fallible<T> {
467    pub fn get(self) -> Result<Arc<T>, InvalidResourceError> {
468        match self {
469            Fallible::Valid(v) => Ok(v),
470            Fallible::Invalid(label) => Err(InvalidResourceError(ResourceErrorIdent {
471                r#type: Cow::Borrowed(T::TYPE),
472                label: (*label).clone(),
473            })),
474        }
475    }
476}
477
478impl<T: ParentDevice> Clone for Fallible<T> {
479    fn clone(&self) -> Self {
480        match self {
481            Self::Valid(v) => Self::Valid(v.clone()),
482            Self::Invalid(l) => Self::Invalid(l.clone()),
483        }
484    }
485}
486
487impl<T: ParentDevice> ResourceType for Fallible<T> {
488    const TYPE: &'static str = T::TYPE;
489}
490
491impl<T: ParentDevice + crate::storage::StorageItem> crate::storage::StorageItem for Fallible<T> {
492    type Marker = T::Marker;
493}
494
495pub type BufferAccessResult = Result<(), BufferAccessError>;
496
497#[derive(Debug)]
498pub(crate) struct BufferPendingMapping {
499    pub(crate) range: Range<wgt::BufferAddress>,
500    pub(crate) op: BufferMapOperation,
501    // hold the parent alive while the mapping is active
502    pub(crate) _parent_buffer: Arc<Buffer>,
503}
504
505pub type BufferDescriptor<'a> = wgt::BufferDescriptor<Label<'a>>;
506
507#[derive(Debug)]
508pub struct Buffer {
509    pub(crate) raw: Snatchable<Box<dyn hal::DynBuffer>>,
510    pub(crate) device: Arc<Device>,
511    pub(crate) usage: wgt::BufferUsages,
512    pub(crate) size: wgt::BufferAddress,
513    pub(crate) initialization_status: RwLock<BufferInitTracker>,
514    /// The `label` from the descriptor used to create the resource.
515    pub(crate) label: String,
516    pub(crate) tracking_data: TrackingData,
517    pub(crate) map_state: Mutex<BufferMapState>,
518    /// What [`Buffer::destroy`] took from a buffer that was mapped. It is freed when the buffer
519    /// drops, since the mapped pointer, and the views over it, read the raw buffer until then.
520    pub(crate) destroyed_while_mapped: crate::OnceCellOrLock<DestroyedBuffer>,
521    // Bind groups that reference this buffer. May contain duplicates.
522    pub(crate) bind_groups: Mutex<WeakVec<BindGroup>>,
523    pub(crate) timestamp_normalization_bind_group: Snatchable<TimestampNormalizationBindGroup>,
524    pub(crate) indirect_validation_bind_groups: Snatchable<crate::indirect_validation::BindGroups>,
525}
526
527impl Drop for Buffer {
528    fn drop(&mut self) {
529        if let Some(raw) = self.timestamp_normalization_bind_group.take() {
530            raw.dispose(self.device.raw());
531        }
532
533        if let Some(raw) = self.indirect_validation_bind_groups.take() {
534            raw.dispose(self.device.raw());
535        }
536
537        if let Some(raw) = self.raw.take() {
538            resource_log!("Destroy raw {}", self.error_ident());
539            unsafe {
540                self.device.raw().destroy_buffer(raw);
541            }
542        }
543    }
544}
545
546impl RawResourceAccess for Buffer {
547    type DynResource = dyn hal::DynBuffer;
548
549    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
550        self.raw.get(guard).map(|b| b.as_ref())
551    }
552}
553
554impl Buffer {
555    pub(crate) fn check_destroyed(
556        &self,
557        guard: &SnatchGuard,
558    ) -> Result<(), DestroyedResourceError> {
559        self.raw
560            .get(guard)
561            .map(|_| ())
562            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
563    }
564
565    /// Checks that the given buffer usage contains the required buffer usage,
566    /// returns an error otherwise.
567    pub(crate) fn check_usage(
568        &self,
569        expected: wgt::BufferUsages,
570    ) -> Result<(), MissingBufferUsageError> {
571        if self.usage.contains(expected) {
572            Ok(())
573        } else {
574            Err(MissingBufferUsageError {
575                res: self.error_ident(),
576                actual: self.usage,
577                expected,
578            })
579        }
580    }
581
582    /// Resolve the size of a binding for buffer with `offset` and `size`.
583    ///
584    /// If `size` is `None`, then the remainder of the buffer starting from
585    /// `offset` is used.
586    ///
587    /// If the binding would overflow the buffer, then an error is returned.
588    ///
589    /// Zero-size bindings are permitted here for historical reasons. Although
590    /// zero-size bindings are permitted by WebGPU, they are not permitted by
591    /// some backends. See [`Buffer::binding`] and
592    /// [#3170](https://github.com/gfx-rs/wgpu/issues/3170).
593    pub fn resolve_binding_size(
594        &self,
595        offset: wgt::BufferAddress,
596        binding_size: Option<wgt::BufferSize>,
597    ) -> Result<u64, BindingError> {
598        let buffer_size = self.size;
599
600        match binding_size {
601            Some(binding_size) => match offset.checked_add(binding_size.get()) {
602                Some(end) if end <= buffer_size => Ok(binding_size.get()),
603                _ => Err(BindingError::BindingRangeTooLarge {
604                    buffer: self.error_ident(),
605                    offset,
606                    binding_size: binding_size.get(),
607                    buffer_size,
608                }),
609            },
610            None => {
611                buffer_size
612                    .checked_sub(offset)
613                    .ok_or_else(|| BindingError::BindingOffsetTooLarge {
614                        buffer: self.error_ident(),
615                        offset,
616                        buffer_size,
617                    })
618            }
619        }
620    }
621
622    /// Create a new [`hal::BufferBinding`] for the buffer with `offset` and
623    /// `binding_size`.
624    ///
625    /// If `binding_size` is `None`, then the remainder of the buffer starting
626    /// from `offset` is used.
627    ///
628    /// If the binding would overflow the buffer, then an error is returned.
629    ///
630    /// A zero-size binding at the end of the buffer is permitted here for historical reasons. Although
631    /// zero-size bindings are permitted by WebGPU, they are not permitted by
632    /// some backends. The zero-size binding need to be quashed or remapped to a
633    /// non-zero size, either universally in wgpu-core, or in specific backends
634    /// that do not support them. See
635    /// [#3170](https://github.com/gfx-rs/wgpu/issues/3170).
636    ///
637    /// Although it seems like it would be simpler and safer to use the resolved
638    /// size in the returned [`hal::BufferBinding`], doing this (and removing
639    /// redundant logic in backends to resolve the implicit size) was observed
640    /// to cause problems in certain CTS tests, so an implicit size
641    /// specification is preserved in the output.
642    pub fn binding<'a>(
643        &'a self,
644        offset: wgt::BufferAddress,
645        binding_size: Option<wgt::BufferSize>,
646        snatch_guard: &'a SnatchGuard,
647    ) -> Result<(hal::BufferBinding<'a, dyn hal::DynBuffer>, u64), BindingError> {
648        let buf_raw = self.try_raw(snatch_guard)?;
649        let resolved_size = self.resolve_binding_size(offset, binding_size)?;
650        // SAFETY: The offset and size passed to hal::BufferBinding::new_unchecked must
651        // define a binding contained within the buffer.
652        Ok((
653            hal::BufferBinding::new_unchecked(buf_raw, offset, binding_size),
654            resolved_size,
655        ))
656    }
657
658    /// Schedule buffer mapping.
659    ///
660    /// `op.callback` is guaranteed to be called, regardless of the outcome.
661    pub fn map_async(
662        self: &Arc<Self>,
663        offset: wgt::BufferAddress,
664        size: Option<wgt::BufferAddress>,
665        op: BufferMapOperation,
666    ) -> Result<SubmissionIndex, BufferAccessError> {
667        self.try_map_async(offset, size, op)
668            .map_err(|(mut operation, err)| {
669                if let Some(callback) = operation.callback.take() {
670                    callback(Err(err.clone()));
671                }
672                err
673            })
674    }
675
676    /// Try to schedule buffer mapping.
677    ///
678    /// The outcome of this function is one of the following:
679    /// - If there is a queue, and nothing pending in the queue that uses the
680    ///   buffer in question, the buffer is added to `Queue::ready_to_map`, and
681    ///   will be mapped the next time `Device::maintain` is called. The
682    ///   queue assumes responsibility for calling the callback, and this
683    ///   function returns `Ok(0)`, but the buffer has not yet been mapped.
684    /// - If there is a queue, and something is pending in the queue that uses
685    ///   the buffer in question, the buffer is scheduled for mapping after that
686    ///   submission completes. The queue assumes responsibility for calling the
687    ///   callback, and this function returns `Ok(index)` with the index of the
688    ///   submission that must complete. The buffer has not yet been mapped.
689    /// - If there is no queue, the buffer is mapped and the callback is called
690    ///   immediately. The return value is `Ok(0)`.
691    /// - Regardless of the queue state, if there is an error that terminates
692    ///   the buffer mapping attempt, this function returns the callback along
693    ///   with the error, and the caller is responsible for calling the
694    ///   callback.
695    ///
696    /// A return value of `Ok(0)` means that mapping does not need to wait on the queue, but
697    /// it does not mean that the buffer has already been mapped.
698    fn try_map_async(
699        self: &Arc<Self>,
700        offset: wgt::BufferAddress,
701        size: Option<wgt::BufferAddress>,
702        op: BufferMapOperation,
703    ) -> Result<SubmissionIndex, (BufferMapOperation, BufferAccessError)> {
704        let range_size = if let Some(size) = size {
705            size
706        } else {
707            self.size.saturating_sub(offset)
708        };
709
710        if !offset.is_multiple_of(wgt::MAP_ALIGNMENT) {
711            return Err((op, BufferAccessError::UnalignedOffset { offset }));
712        }
713        if !range_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
714            return Err((op, BufferAccessError::UnalignedRangeSize { range_size }));
715        }
716
717        if offset > self.size {
718            return Err((
719                op,
720                BufferAccessError::MapStartOffsetOverrun {
721                    offset,
722                    buffer_size: self.size,
723                },
724            ));
725        }
726        // NOTE: Should never underflow because of our earlier check.
727        if range_size > self.size - offset {
728            return Err((
729                op,
730                BufferAccessError::MapEndOffsetOverrun {
731                    offset,
732                    size: range_size,
733                    buffer_size: self.size,
734                },
735            ));
736        }
737        let end_offset = offset + range_size;
738
739        if !offset.is_multiple_of(wgt::MAP_ALIGNMENT)
740            || !end_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT)
741        {
742            return Err((op, BufferAccessError::UnalignedRange));
743        }
744
745        let (pub_usage, internal_use) = match op.host {
746            HostMap::Read => (wgt::BufferUsages::MAP_READ, wgt::BufferUses::MAP_READ),
747            HostMap::Write => (wgt::BufferUsages::MAP_WRITE, wgt::BufferUses::MAP_WRITE),
748        };
749
750        if let Err(e) = self.check_usage(pub_usage) {
751            return Err((op, e.into()));
752        }
753
754        let device = &self.device;
755        if let Err(e) = device.check_is_valid() {
756            return Err((op, e.into()));
757        }
758
759        let submit_index = {
760            let snatch_guard = device.snatchable_lock.read();
761            if let Err(e) = self.check_destroyed(&snatch_guard) {
762                return Err((op, e.into()));
763            }
764
765            {
766                let map_state = &mut *self.map_state.lock();
767                *map_state = match *map_state {
768                    BufferMapState::Init { .. } | BufferMapState::Active { .. } => {
769                        return Err((op, BufferAccessError::AlreadyMapped));
770                    }
771                    BufferMapState::Waiting(_) => {
772                        return Err((op, BufferAccessError::MapAlreadyPending));
773                    }
774                    BufferMapState::Idle => BufferMapState::Waiting(BufferPendingMapping {
775                        range: offset..end_offset,
776                        op,
777                        _parent_buffer: self.clone(),
778                    }),
779                };
780            }
781
782            if let Some(queue) = device.get_queue().as_ref() {
783                match queue.flush_writes_for_buffer(self, snatch_guard) {
784                    Err(err) => {
785                        let state = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle);
786                        let BufferMapState::Waiting(BufferPendingMapping { op, .. }) = state else {
787                            unreachable!();
788                        };
789                        return Err((op, err));
790                    }
791                    Ok(()) => {
792                        // Schedule the buffer map in the  lifetime tracker.
793                        //
794                        // This call searches for use of the buffer by pending submissions.
795                        // If we just flushed pending writes, that search is redundant; we
796                        // already know that mapping needs to wait for the latest submission
797                        // and could implement a special case to directly attach it to that
798                        // submission. However, the queue is searched in reverse, so finding
799                        // that the buffer is used by the latest submission will be fast.
800                        Some(queue.lock_life().map(self).unwrap_or(0))
801                    }
802                }
803            } else {
804                None
805            }
806        };
807
808        // At this point, `submit_index` is:
809        // - `Some(index)`, if there is a submission the mapping operation must wait for.
810        // - `Some(0)`, if we have a queue and there is no submission to wait for.
811        // - `None`, if we don't have a queue.
812        //
813        // TODO(https://github.com/gfx-rs/wgpu/issues/9306): we are ignoring the transition
814        // here, I think we need to add a barrier at the end of the submission
815        device
816            .trackers
817            .lock()
818            .buffers
819            .set_single(self, internal_use);
820
821        if let Some(index) = submit_index {
822            Ok(index)
823        } else {
824            // We don't have a queue, so go ahead and map the buffer.
825            // We can safely unwrap below since we just set the `map_state` to `BufferMapState::Waiting`.
826            let (mut operation, status) = self.map(&device.snatchable_lock.read()).unwrap();
827            if let Some(callback) = operation.callback.take() {
828                callback(status);
829            }
830            Ok(0)
831        }
832    }
833
834    pub fn get_mapped_range(
835        self: &Arc<Self>,
836        offset: wgt::BufferAddress,
837        size: Option<wgt::BufferAddress>,
838    ) -> Result<(NonNull<u8>, u64), BufferAccessError> {
839        {
840            let snatch_guard = self.device.snatchable_lock.read();
841            self.check_destroyed(&snatch_guard)?;
842        }
843
844        let range_size = if let Some(size) = size {
845            size
846        } else {
847            self.size.saturating_sub(offset)
848        };
849
850        if !offset.is_multiple_of(wgt::MAP_ALIGNMENT) {
851            return Err(BufferAccessError::UnalignedOffset { offset });
852        }
853        if !range_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
854            return Err(BufferAccessError::UnalignedRangeSize { range_size });
855        }
856        let map_state = &*self.map_state.lock();
857        match *map_state {
858            BufferMapState::Init { ref staging_buffer } => {
859                if offset > self.size {
860                    return Err(BufferAccessError::MapStartOffsetOverrun {
861                        offset,
862                        buffer_size: self.size,
863                    });
864                }
865                // NOTE: Should never underflow because of our earlier check.
866                if range_size > self.size - offset {
867                    return Err(BufferAccessError::MapEndOffsetOverrun {
868                        offset,
869                        size: range_size,
870                        buffer_size: self.size,
871                    });
872                }
873                let ptr = unsafe { staging_buffer.ptr() };
874                let ptr = unsafe { NonNull::new_unchecked(ptr.as_ptr().offset(offset as isize)) };
875                Ok((ptr, range_size))
876            }
877            BufferMapState::Active {
878                ref mapping,
879                ref range,
880                ..
881            } => {
882                if offset > range.end {
883                    return Err(BufferAccessError::OutOfBoundsStartOffsetOverrun {
884                        index: offset,
885                        max: range.end,
886                    });
887                }
888                if offset < range.start {
889                    return Err(BufferAccessError::OutOfBoundsStartOffsetUnderrun {
890                        index: offset,
891                        min: range.start,
892                    });
893                }
894                if range_size > range.end - offset {
895                    return Err(BufferAccessError::OutOfBoundsEndOffsetOverrun {
896                        index: offset,
897                        size: range_size,
898                        max: range.end,
899                    });
900                }
901                // ptr points to the beginning of the range we mapped in map_async
902                // rather than the beginning of the buffer.
903                let relative_offset = (offset - range.start) as isize;
904                unsafe {
905                    Ok((
906                        NonNull::new_unchecked(mapping.ptr.as_ptr().offset(relative_offset)),
907                        range_size,
908                    ))
909                }
910            }
911            BufferMapState::Idle | BufferMapState::Waiting(_) => Err(BufferAccessError::NotMapped),
912        }
913    }
914    /// This function returns [`None`] only if [`Self::map_state`] is not [`BufferMapState::Waiting`].
915    /// Other errors are returned within `BufferMapPendingClosure`.
916    #[must_use]
917    pub(crate) fn map(&self, snatch_guard: &SnatchGuard) -> Option<BufferMapPendingClosure> {
918        // This _cannot_ be inlined into the match. If it is, the lock will be held
919        // open through the whole match, resulting in a deadlock when we try to re-lock
920        // the buffer back to active.
921        let mapping = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle);
922        let pending_mapping = match mapping {
923            BufferMapState::Waiting(pending_mapping) => pending_mapping,
924            // Mapping cancelled
925            BufferMapState::Idle => return None,
926            // Mapping queued at least twice by map -> unmap -> map
927            // and was already successfully mapped below
928            BufferMapState::Active { .. } => {
929                *self.map_state.lock() = mapping;
930                return None;
931            }
932            _ => panic!("No pending mapping."),
933        };
934        let status = if pending_mapping.range.start != pending_mapping.range.end {
935            let host = pending_mapping.op.host;
936            let size = pending_mapping.range.end - pending_mapping.range.start;
937            match crate::device::map_buffer(
938                self,
939                pending_mapping.range.start,
940                size,
941                host,
942                snatch_guard,
943            ) {
944                Ok(mapping) => {
945                    *self.map_state.lock() = BufferMapState::Active {
946                        mapping,
947                        range: pending_mapping.range.clone(),
948                        host,
949                    };
950                    Ok(())
951                }
952                Err(e) => Err(e),
953            }
954        } else {
955            *self.map_state.lock() = BufferMapState::Active {
956                mapping: hal::BufferMapping {
957                    ptr: NonNull::dangling(),
958                    is_coherent: true,
959                },
960                range: pending_mapping.range,
961                host: pending_mapping.op.host,
962            };
963            Ok(())
964        };
965        Some((pending_mapping.op, status))
966    }
967
968    // Note: This must not be called while holding a lock.
969    pub fn unmap(self: &Arc<Self>) -> Result<(), BufferAccessError> {
970        if let Some((mut operation, status)) = self.unmap_inner()? {
971            if let Some(callback) = operation.callback.take() {
972                callback(status);
973            }
974        }
975
976        Ok(())
977    }
978
979    fn unmap_inner(self: &Arc<Self>) -> Result<Option<BufferMapPendingClosure>, BufferAccessError> {
980        let device = &self.device;
981        let snatch_guard = device.snatchable_lock.read();
982        let raw_buf = self.try_raw(&snatch_guard)?;
983        let map_state = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle);
984        match map_state {
985            BufferMapState::Init { staging_buffer } => {
986                #[cfg(feature = "trace")]
987                if let Some(ref mut trace) = *device.trace.lock() {
988                    use crate::device::trace::{DataKind, IntoTrace};
989
990                    let data = trace.make_binary(DataKind::Bin, staging_buffer.get_data());
991                    trace.add(trace::Action::WriteBuffer {
992                        id: self.to_trace(),
993                        data,
994                        // NOTE: `self.size` here corresponds to `data`'s actual length.
995                        offset: 0,
996                        size: self.size,
997                        queued: true,
998                    });
999                }
1000
1001                let staging_buffer = staging_buffer.flush();
1002
1003                if let Some(queue) = device.get_queue() {
1004                    let region = wgt::BufferSize::new(self.size).map(|size| hal::BufferCopy {
1005                        src_offset: 0,
1006                        dst_offset: 0,
1007                        size,
1008                    });
1009                    let transition_src = hal::BufferBarrier {
1010                        buffer: staging_buffer.raw(),
1011                        usage: hal::StateTransition {
1012                            from: wgt::BufferUses::MAP_WRITE,
1013                            to: wgt::BufferUses::COPY_SRC,
1014                        },
1015                    };
1016                    let transition_dst = hal::BufferBarrier::<dyn hal::DynBuffer> {
1017                        buffer: raw_buf,
1018                        usage: hal::StateTransition {
1019                            from: wgt::BufferUses::empty(),
1020                            to: wgt::BufferUses::COPY_DST,
1021                        },
1022                    };
1023                    let mut pending_writes = queue.pending_writes.lock();
1024                    let encoder = pending_writes.activate();
1025                    unsafe {
1026                        encoder.transition_buffers(&[transition_src, transition_dst]);
1027                        if self.size > 0 {
1028                            encoder.copy_buffer_to_buffer(
1029                                staging_buffer.raw(),
1030                                raw_buf,
1031                                region.as_slice(),
1032                            );
1033                        }
1034                    }
1035                    pending_writes.consume(staging_buffer);
1036                    pending_writes.insert_buffer(self);
1037                }
1038            }
1039            BufferMapState::Idle => {
1040                return Err(BufferAccessError::NotMapped);
1041            }
1042            BufferMapState::Waiting(pending) => {
1043                return Ok(Some((pending.op, Err(BufferAccessError::MapAborted))));
1044            }
1045            BufferMapState::Active {
1046                mapping,
1047                range,
1048                host,
1049            } => {
1050                if host == HostMap::Write {
1051                    #[cfg(feature = "trace")]
1052                    if let Some(ref mut trace) = *device.trace.lock() {
1053                        use crate::device::trace::{DataKind, IntoTrace};
1054
1055                        let size = range.end - range.start;
1056                        let data = trace.make_binary(DataKind::Bin, unsafe {
1057                            core::slice::from_raw_parts(mapping.ptr.as_ptr(), size as usize)
1058                        });
1059                        trace.add(trace::Action::WriteBuffer {
1060                            id: self.to_trace(),
1061                            data,
1062                            offset: range.start,
1063                            size,
1064                            queued: false,
1065                        });
1066                    }
1067                    if !mapping.is_coherent {
1068                        unsafe { device.raw().flush_mapped_ranges(raw_buf, &[range]) };
1069                    }
1070                }
1071                unsafe { device.raw().unmap_buffer(raw_buf) };
1072            }
1073        }
1074        Ok(None)
1075    }
1076
1077    pub fn destroy(self: &Arc<Self>) {
1078        let device = &self.device;
1079
1080        let destroyed = {
1081            let mut snatch_guard = device.snatchable_lock.write();
1082
1083            let raw = match self.raw.snatch(&mut snatch_guard) {
1084                Some(raw) => raw,
1085                None => {
1086                    // Per spec, it is valid to call `destroy` multiple times.
1087                    return;
1088                }
1089            };
1090
1091            let timestamp_normalization_bind_group = self
1092                .timestamp_normalization_bind_group
1093                .snatch(&mut snatch_guard);
1094
1095            let indirect_validation_bind_groups = self
1096                .indirect_validation_bind_groups
1097                .snatch(&mut snatch_guard);
1098
1099            drop(snatch_guard);
1100
1101            let bind_groups = {
1102                let mut guard = self.bind_groups.lock();
1103                mem::take(&mut *guard)
1104            };
1105
1106            DestroyedBuffer {
1107                raw: ManuallyDrop::new(raw),
1108                device: Arc::clone(&self.device),
1109                label: self.label().to_owned(),
1110                bind_groups,
1111                timestamp_normalization_bind_group,
1112                indirect_validation_bind_groups,
1113            }
1114        };
1115
1116        if matches!(*self.map_state.lock(), BufferMapState::Active { .. }) {
1117            let _ = self.destroyed_while_mapped.set(destroyed);
1118            return;
1119        }
1120        let temp = queue::TempResource::DestroyedBuffer(destroyed);
1121
1122        let Some(queue) = device.get_queue() else {
1123            return;
1124        };
1125
1126        {
1127            let mut pending_writes = queue.pending_writes.lock();
1128            if pending_writes.contains_buffer(self) {
1129                pending_writes.consume_temp(temp);
1130                return;
1131            }
1132        }
1133
1134        let mut life_lock = queue.lock_life();
1135        let last_submit_index = life_lock.get_buffer_latest_submission_index(self);
1136        if let Some(last_submit_index) = last_submit_index {
1137            life_lock.schedule_resource_destruction(temp, last_submit_index);
1138        }
1139    }
1140}
1141
1142#[derive(Clone, Debug, Error)]
1143#[non_exhaustive]
1144pub enum CreateBufferError {
1145    #[error(transparent)]
1146    Device(#[from] DeviceError),
1147    #[error("Failed to map buffer while creating: {0}")]
1148    AccessError(#[from] BufferAccessError),
1149    #[error("Buffers that are mapped at creation have to be aligned to `COPY_BUFFER_ALIGNMENT`")]
1150    UnalignedSize,
1151    #[error("Invalid usage flags {0:?}")]
1152    InvalidUsage(wgt::BufferUsages),
1153    #[error("`MAP` usage can only be combined with the opposite `COPY`, requested {0:?}")]
1154    UsageMismatch(wgt::BufferUsages),
1155    #[error("Buffer size {requested} is greater than the maximum buffer size ({maximum})")]
1156    MaxBufferSize { requested: u64, maximum: u64 },
1157    #[error(transparent)]
1158    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
1159    #[error(transparent)]
1160    MissingFeatures(#[from] MissingFeatures),
1161    #[error("Failed to create bind group for indirect buffer validation: {0}")]
1162    IndirectValidationBindGroup(DeviceError),
1163}
1164
1165crate::impl_resource_type!(Buffer);
1166crate::impl_labeled!(Buffer);
1167crate::impl_parent_device!(Buffer);
1168crate::impl_storage_item!(Buffer);
1169crate::impl_trackable!(Buffer);
1170
1171impl WebGpuError for CreateBufferError {
1172    fn webgpu_error_type(&self) -> ErrorType {
1173        match self {
1174            Self::Device(e) => e.webgpu_error_type(),
1175            Self::AccessError(e) => e.webgpu_error_type(),
1176            Self::MissingDownlevelFlags(e) => e.webgpu_error_type(),
1177            Self::IndirectValidationBindGroup(e) => e.webgpu_error_type(),
1178            Self::MissingFeatures(e) => e.webgpu_error_type(),
1179
1180            Self::UnalignedSize
1181            | Self::InvalidUsage(_)
1182            | Self::UsageMismatch(_)
1183            | Self::MaxBufferSize { .. } => ErrorType::Validation,
1184        }
1185    }
1186}
1187
1188/// A buffer that has been marked as destroyed and is staged for actual deletion soon.
1189#[derive(Debug)]
1190pub struct DestroyedBuffer {
1191    raw: ManuallyDrop<Box<dyn hal::DynBuffer>>,
1192    device: Arc<Device>,
1193    label: String,
1194    bind_groups: WeakVec<BindGroup>,
1195    timestamp_normalization_bind_group: Option<TimestampNormalizationBindGroup>,
1196    indirect_validation_bind_groups: Option<crate::indirect_validation::BindGroups>,
1197}
1198
1199impl DestroyedBuffer {
1200    pub fn label(&self) -> &dyn fmt::Debug {
1201        &self.label
1202    }
1203}
1204
1205impl Drop for DestroyedBuffer {
1206    fn drop(&mut self) {
1207        let mut deferred = self.device.deferred_destroy.lock();
1208        deferred.push(DeferredDestroy::BindGroups(mem::take(
1209            &mut self.bind_groups,
1210        )));
1211        drop(deferred);
1212
1213        if let Some(raw) = self.timestamp_normalization_bind_group.take() {
1214            raw.dispose(self.device.raw());
1215        }
1216
1217        if let Some(raw) = self.indirect_validation_bind_groups.take() {
1218            raw.dispose(self.device.raw());
1219        }
1220
1221        resource_log!("Destroy raw Buffer (destroyed) {:?}", self.label());
1222        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
1223        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1224        unsafe {
1225            hal::DynDevice::destroy_buffer(self.device.raw(), raw);
1226        }
1227    }
1228}
1229
1230#[cfg(send_sync)]
1231unsafe impl Send for StagingBuffer {}
1232#[cfg(send_sync)]
1233unsafe impl Sync for StagingBuffer {}
1234
1235/// A temporary buffer, consumed by the command that uses it.
1236///
1237/// A [`StagingBuffer`] is designed for one-shot uploads of data to the GPU. It
1238/// is always created mapped, and the command that uses it destroys the buffer
1239/// when it is done.
1240///
1241/// [`StagingBuffer`]s can be created with [`queue_create_staging_buffer`] and
1242/// used with [`queue_write_staging_buffer`]. They are also used internally by
1243/// operations like [`queue_write_texture`] that need to upload data to the GPU,
1244/// but that don't belong to any particular wgpu command buffer.
1245///
1246/// Used `StagingBuffer`s are accumulated in [`Device::pending_writes`], to be
1247/// freed once their associated operation's queue submission has finished
1248/// execution.
1249///
1250/// [`queue_create_staging_buffer`]: crate::global::Global::queue_create_staging_buffer
1251/// [`queue_write_staging_buffer`]: crate::global::Global::queue_write_staging_buffer
1252/// [`queue_write_texture`]: crate::global::Global::queue_write_texture
1253/// [`Device::pending_writes`]: crate::device::Device
1254#[derive(Debug)]
1255pub struct StagingBuffer {
1256    raw: Box<dyn hal::DynBuffer>,
1257    device: Arc<Device>,
1258    pub(crate) size: wgt::BufferSize,
1259    is_coherent: bool,
1260    ptr: NonNull<u8>,
1261}
1262
1263impl StagingBuffer {
1264    pub(crate) fn new(device: &Arc<Device>, size: wgt::BufferSize) -> Result<Self, DeviceError> {
1265        profiling::scope!("StagingBuffer::new");
1266        let stage_desc = hal::BufferDescriptor {
1267            label: hal_label(Some("(wgpu internal) Staging"), device.instance_flags),
1268            size: size.get(),
1269            usage: wgt::BufferUses::MAP_WRITE | wgt::BufferUses::COPY_SRC,
1270            memory_flags: hal::MemoryFlags::TRANSIENT,
1271        };
1272
1273        let raw = unsafe { device.raw().create_buffer(&stage_desc) }
1274            .map_err(|e| device.handle_hal_error(e))?;
1275        let mapping = unsafe { device.raw().map_buffer(raw.as_ref(), 0..size.get()) }
1276            .map_err(|e| device.handle_hal_error(e))?;
1277
1278        let staging_buffer = StagingBuffer {
1279            raw,
1280            device: device.clone(),
1281            size,
1282            is_coherent: mapping.is_coherent,
1283            ptr: mapping.ptr,
1284        };
1285
1286        Ok(staging_buffer)
1287    }
1288
1289    /// SAFETY: You must not call any functions of `self`
1290    /// until you stopped using the returned pointer.
1291    pub(crate) unsafe fn ptr(&self) -> NonNull<u8> {
1292        self.ptr
1293    }
1294
1295    #[cfg(feature = "trace")]
1296    pub(crate) fn get_data(&self) -> &[u8] {
1297        unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.size.get() as usize) }
1298    }
1299
1300    pub(crate) fn write_zeros(&mut self) {
1301        unsafe { core::ptr::write_bytes(self.ptr.as_ptr(), 0, self.size.get() as usize) };
1302    }
1303
1304    pub(crate) fn write(&mut self, data: &[u8]) {
1305        assert!(data.len() >= self.size.get() as usize);
1306        // SAFETY: With the assert above, all of `copy_nonoverlapping`'s
1307        // requirements are satisfied.
1308        unsafe {
1309            core::ptr::copy_nonoverlapping(
1310                data.as_ptr(),
1311                self.ptr.as_ptr(),
1312                self.size.get() as usize,
1313            );
1314        }
1315    }
1316
1317    /// SAFETY: The offsets and size must be in-bounds.
1318    pub(crate) unsafe fn write_with_offset(
1319        &mut self,
1320        data: &[u8],
1321        src_offset: isize,
1322        dst_offset: isize,
1323        size: usize,
1324    ) {
1325        unsafe {
1326            debug_assert!(
1327                (src_offset + size as isize) as usize <= data.len(),
1328                "src_offset + size must be in-bounds: src_offset = {}, size = {}, data.len() = {}",
1329                src_offset,
1330                size,
1331                data.len()
1332            );
1333            core::ptr::copy_nonoverlapping(
1334                data.as_ptr().offset(src_offset),
1335                self.ptr.as_ptr().offset(dst_offset),
1336                size,
1337            );
1338        }
1339    }
1340
1341    pub(crate) fn flush(self) -> FlushedStagingBuffer {
1342        let device = self.device.raw();
1343        if !self.is_coherent {
1344            #[allow(clippy::single_range_in_vec_init)]
1345            unsafe {
1346                device.flush_mapped_ranges(self.raw.as_ref(), &[0..self.size.get()])
1347            };
1348        }
1349        unsafe { device.unmap_buffer(self.raw.as_ref()) };
1350
1351        let StagingBuffer {
1352            raw, device, size, ..
1353        } = self;
1354
1355        FlushedStagingBuffer {
1356            raw: ManuallyDrop::new(raw),
1357            device,
1358            size,
1359        }
1360    }
1361}
1362
1363crate::impl_resource_type!(StagingBuffer);
1364crate::impl_storage_item!(StagingBuffer);
1365
1366#[derive(Debug)]
1367pub struct FlushedStagingBuffer {
1368    raw: ManuallyDrop<Box<dyn hal::DynBuffer>>,
1369    device: Arc<Device>,
1370    pub(crate) size: wgt::BufferSize,
1371}
1372
1373impl FlushedStagingBuffer {
1374    pub(crate) fn raw(&self) -> &dyn hal::DynBuffer {
1375        self.raw.as_ref()
1376    }
1377}
1378
1379impl Drop for FlushedStagingBuffer {
1380    fn drop(&mut self) {
1381        resource_log!("Destroy raw StagingBuffer");
1382        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
1383        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1384        unsafe { self.device.raw().destroy_buffer(raw) };
1385    }
1386}
1387
1388pub type TextureDescriptor<'a> = wgt::TextureDescriptor<Label<'a>, Vec<wgt::TextureFormat>>;
1389
1390#[derive(Debug)]
1391pub(crate) enum TextureInner {
1392    Native {
1393        raw: Box<dyn hal::DynTexture>,
1394    },
1395    Surface {
1396        raw: Box<dyn hal::DynSurfaceTexture>,
1397    },
1398}
1399
1400impl TextureInner {
1401    pub(crate) fn raw(&self) -> &dyn hal::DynTexture {
1402        match self {
1403            Self::Native { raw } => raw.as_ref(),
1404            Self::Surface { raw, .. } => raw.as_ref().borrow(),
1405        }
1406    }
1407}
1408
1409#[derive(Debug)]
1410pub enum TextureClearMode {
1411    BufferCopy,
1412    // View for clear via RenderPass for every subsurface (mip/layer/slice)
1413    RenderPass {
1414        clear_views: SmallVec<[ManuallyDrop<Box<dyn hal::DynTextureView>>; 1]>,
1415        is_color: bool,
1416    },
1417    Surface {
1418        clear_view: ManuallyDrop<Box<dyn hal::DynTextureView>>,
1419    },
1420    // Texture can't be cleared, attempting to do so will cause panic.
1421    // (either because it is impossible for the type of texture or it is being destroyed)
1422    None,
1423}
1424
1425#[derive(Debug)]
1426pub struct Texture {
1427    pub(crate) inner: Snatchable2<TextureInner>,
1428    pub(crate) device: Arc<Device>,
1429    pub(crate) desc: wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
1430    pub(crate) _hal_usage: wgt::TextureUses,
1431    pub(crate) format_features: wgt::TextureFormatFeatures,
1432    pub(crate) initialization_status: RwLock<TextureInitTracker>,
1433    pub(crate) full_range: TextureSelector,
1434    pub(crate) tracking_data: TrackingData,
1435    pub(crate) clear_mode: RwLock<TextureClearMode>,
1436    pub(crate) views: Mutex<WeakVec<TextureView>>,
1437    // Bind groups that reference this texture. May contain duplicates.
1438    pub(crate) bind_groups: Mutex<WeakVec<BindGroup>>,
1439}
1440
1441impl Texture {
1442    pub(crate) fn new(
1443        device: &Arc<Device>,
1444        inner: TextureInner,
1445        hal_usage: wgt::TextureUses,
1446        desc: &TextureDescriptor,
1447        format_features: wgt::TextureFormatFeatures,
1448        clear_mode: TextureClearMode,
1449        init: bool,
1450    ) -> Self {
1451        Texture {
1452            inner: Snatchable2::new(inner),
1453            device: device.clone(),
1454            desc: desc.map_label(|label| label.to_string()),
1455            _hal_usage: hal_usage,
1456            format_features,
1457            initialization_status: RwLock::new(
1458                rank::TEXTURE_INITIALIZATION_STATUS,
1459                if init {
1460                    TextureInitTracker::new(desc.mip_level_count, desc.array_layer_count())
1461                } else {
1462                    TextureInitTracker::new(desc.mip_level_count, 0)
1463                },
1464            ),
1465            full_range: TextureSelector {
1466                mips: 0..desc.mip_level_count,
1467                layers: 0..desc.array_layer_count(),
1468            },
1469            tracking_data: TrackingData::new(device.tracker_indices.textures.clone()),
1470            clear_mode: RwLock::new(rank::TEXTURE_CLEAR_MODE, clear_mode),
1471            views: Mutex::new(rank::TEXTURE_VIEWS, WeakVec::new()),
1472            bind_groups: Mutex::new(rank::TEXTURE_BIND_GROUPS, WeakVec::new()),
1473        }
1474    }
1475
1476    pub(crate) fn invalid(device: &Arc<Device>, desc: &TextureDescriptor) -> Self {
1477        Texture {
1478            inner: Snatchable2::invalid(),
1479            device: device.clone(),
1480            desc: desc.map_label(|label| label.to_string()),
1481            _hal_usage: wgt::TextureUses::empty(),
1482            format_features: wgt::TextureFormatFeatures {
1483                allowed_usages: wgt::TextureUsages::empty(),
1484                flags: wgt::TextureFormatFeatureFlags::empty(),
1485            },
1486            initialization_status: RwLock::new(
1487                rank::TEXTURE_INITIALIZATION_STATUS,
1488                TextureInitTracker::new(0, 0),
1489            ),
1490            full_range: TextureSelector {
1491                mips: 0..desc.mip_level_count,
1492                layers: 0..desc.array_layer_count(),
1493            },
1494            tracking_data: TrackingData::new(device.tracker_indices.textures.clone()),
1495            clear_mode: RwLock::new(rank::TEXTURE_CLEAR_MODE, TextureClearMode::None),
1496            views: Mutex::new(rank::TEXTURE_VIEWS, WeakVec::new()),
1497            bind_groups: Mutex::new(rank::TEXTURE_BIND_GROUPS, WeakVec::new()),
1498        }
1499    }
1500
1501    /// Checks that the given texture usage contains the required texture usage,
1502    /// returns an error otherwise.
1503    pub(crate) fn check_usage(
1504        &self,
1505        expected: wgt::TextureUsages,
1506    ) -> Result<(), MissingTextureUsageError> {
1507        if self.desc.usage.contains(expected) {
1508            Ok(())
1509        } else {
1510            Err(MissingTextureUsageError {
1511                res: self.error_ident(),
1512                actual: self.desc.usage,
1513                expected,
1514            })
1515        }
1516    }
1517}
1518
1519impl Drop for Texture {
1520    fn drop(&mut self) {
1521        #[cfg(feature = "trace")]
1522        {
1523            let mut t = self.device.trace.lock();
1524            if let Some(t) = t.as_mut() {
1525                use crate::device::trace::to_trace;
1526
1527                // SAFETY: All textures are constructed in Arc => are heap allocated
1528                t.add(trace::Action::DropTexture(unsafe { to_trace(self) }));
1529            }
1530        }
1531        match *self.clear_mode.write() {
1532            TextureClearMode::Surface {
1533                ref mut clear_view, ..
1534            } => {
1535                // SAFETY: We are in the Drop impl and we don't use clear_view anymore after this point.
1536                let raw = unsafe { ManuallyDrop::take(clear_view) };
1537                unsafe {
1538                    self.device.raw().destroy_texture_view(raw);
1539                }
1540            }
1541            TextureClearMode::RenderPass {
1542                ref mut clear_views,
1543                ..
1544            } => {
1545                clear_views.iter_mut().for_each(|clear_view| {
1546                    // SAFETY: We are in the Drop impl and we don't use clear_view anymore after this point.
1547                    let raw = unsafe { ManuallyDrop::take(clear_view) };
1548                    unsafe {
1549                        self.device.raw().destroy_texture_view(raw);
1550                    }
1551                });
1552            }
1553            _ => {}
1554        };
1555
1556        if let Some(TextureInner::Native { raw }) = self.inner.take().maybe_valid() {
1557            resource_log!("Destroy raw {}", self.error_ident());
1558            unsafe {
1559                self.device.raw().destroy_texture(raw);
1560            }
1561        }
1562    }
1563}
1564
1565impl RawResourceAccess for Texture {
1566    type DynResource = dyn hal::DynTexture;
1567
1568    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
1569        self.inner.get(guard).maybe_valid().map(|t| t.raw())
1570    }
1571}
1572
1573impl Texture {
1574    pub(crate) fn try_inner<'a>(
1575        &'a self,
1576        guard: &'a SnatchGuard,
1577    ) -> Result<&'a TextureInner, InvalidOrDestroyedResourceError> {
1578        match self.inner.get(guard) {
1579            DestructibleResourceState::Valid(t) => Ok(t),
1580            DestructibleResourceState::Invalid => {
1581                Err(InvalidOrDestroyedResourceError::InvalidResource(
1582                    InvalidResourceError(self.error_ident()),
1583                ))
1584            }
1585            DestructibleResourceState::Destroyed => {
1586                Err(InvalidOrDestroyedResourceError::DestroyedResource(
1587                    DestroyedResourceError(self.error_ident()),
1588                ))
1589            }
1590        }
1591    }
1592
1593    pub(crate) fn check_destroyed(
1594        &self,
1595        guard: &SnatchGuard,
1596    ) -> Result<(), DestroyedResourceError> {
1597        match self.inner.get(guard) {
1598            DestructibleResourceState::Valid(_) => Ok(()),
1599            DestructibleResourceState::Invalid => Ok(()),
1600            DestructibleResourceState::Destroyed => Err(DestroyedResourceError(self.error_ident())),
1601        }
1602    }
1603
1604    pub(crate) fn check_valid(&self, guard: &SnatchGuard) -> Result<(), InvalidResourceError> {
1605        match self.inner.get(guard) {
1606            DestructibleResourceState::Valid(_) => Ok(()),
1607            DestructibleResourceState::Invalid => Err(InvalidResourceError(self.error_ident())),
1608            DestructibleResourceState::Destroyed => Ok(()),
1609        }
1610    }
1611
1612    pub(crate) fn get_clear_view<'a>(
1613        clear_mode: &'a TextureClearMode,
1614        desc: &'a wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
1615        mip_level: u32,
1616        depth_or_layer: u32,
1617    ) -> &'a dyn hal::DynTextureView {
1618        match *clear_mode {
1619            TextureClearMode::BufferCopy => {
1620                panic!("Given texture is cleared with buffer copies, not render passes")
1621            }
1622            TextureClearMode::None => {
1623                panic!("Given texture can't be cleared")
1624            }
1625            TextureClearMode::Surface { ref clear_view, .. } => clear_view.as_ref(),
1626            TextureClearMode::RenderPass {
1627                ref clear_views, ..
1628            } => {
1629                let index = if desc.dimension == wgt::TextureDimension::D3 {
1630                    (0..mip_level).fold(0, |acc, mip| {
1631                        acc + (desc.size.depth_or_array_layers >> mip).max(1)
1632                    })
1633                } else {
1634                    mip_level * desc.size.depth_or_array_layers
1635                } + depth_or_layer;
1636                clear_views[index as usize].as_ref()
1637            }
1638        }
1639    }
1640
1641    pub fn destroy(self: &Arc<Self>) {
1642        let device = &self.device;
1643
1644        let temp = {
1645            let raw = match self
1646                .inner
1647                .snatch(&mut device.snatchable_lock.write())
1648                .maybe_valid()
1649            {
1650                Some(TextureInner::Native { raw }) => raw,
1651                Some(TextureInner::Surface { .. }) => {
1652                    return;
1653                }
1654                None => {
1655                    // Per spec, it is valid to call `destroy` multiple times.
1656                    return;
1657                }
1658            };
1659
1660            let views = {
1661                let mut guard = self.views.lock();
1662                mem::take(&mut *guard)
1663            };
1664
1665            let bind_groups = {
1666                let mut guard = self.bind_groups.lock();
1667                mem::take(&mut *guard)
1668            };
1669
1670            queue::TempResource::DestroyedTexture(DestroyedTexture {
1671                raw: ManuallyDrop::new(raw),
1672                views,
1673                clear_mode: mem::replace(&mut *self.clear_mode.write(), TextureClearMode::None),
1674                bind_groups,
1675                device: Arc::clone(&self.device),
1676                label: self.label().to_owned(),
1677            })
1678        };
1679
1680        let Some(queue) = device.get_queue() else {
1681            return;
1682        };
1683
1684        {
1685            let mut pending_writes = queue.pending_writes.lock();
1686            if pending_writes.contains_texture(self) {
1687                pending_writes.consume_temp(temp);
1688                return;
1689            }
1690        }
1691
1692        let mut life_lock = queue.lock_life();
1693        let last_submit_index = life_lock.get_texture_latest_submission_index(self);
1694        if let Some(last_submit_index) = last_submit_index {
1695            life_lock.schedule_resource_destruction(temp, last_submit_index);
1696        }
1697    }
1698}
1699
1700/// A texture that has been marked as destroyed and is staged for actual deletion soon.
1701#[derive(Debug)]
1702pub struct DestroyedTexture {
1703    raw: ManuallyDrop<Box<dyn hal::DynTexture>>,
1704    views: WeakVec<TextureView>,
1705    clear_mode: TextureClearMode,
1706    bind_groups: WeakVec<BindGroup>,
1707    device: Arc<Device>,
1708    label: String,
1709}
1710
1711impl DestroyedTexture {
1712    pub fn label(&self) -> &dyn fmt::Debug {
1713        &self.label
1714    }
1715}
1716
1717impl Drop for DestroyedTexture {
1718    fn drop(&mut self) {
1719        let device = &self.device;
1720
1721        let mut deferred = device.deferred_destroy.lock();
1722        deferred.push(DeferredDestroy::TextureViews(mem::take(&mut self.views)));
1723        deferred.push(DeferredDestroy::BindGroups(mem::take(
1724            &mut self.bind_groups,
1725        )));
1726        drop(deferred);
1727
1728        match mem::replace(&mut self.clear_mode, TextureClearMode::None) {
1729            TextureClearMode::RenderPass { clear_views, .. } => {
1730                for clear_view in clear_views {
1731                    let raw = ManuallyDrop::into_inner(clear_view);
1732                    unsafe { self.device.raw().destroy_texture_view(raw) };
1733                }
1734            }
1735            TextureClearMode::Surface { clear_view } => {
1736                let raw = ManuallyDrop::into_inner(clear_view);
1737                unsafe { self.device.raw().destroy_texture_view(raw) };
1738            }
1739            _ => (),
1740        }
1741
1742        resource_log!("Destroy raw Texture (destroyed) {:?}", self.label());
1743        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
1744        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1745        unsafe {
1746            self.device.raw().destroy_texture(raw);
1747        }
1748    }
1749}
1750
1751#[derive(Clone, Copy, Debug)]
1752pub enum TextureErrorDimension {
1753    X,
1754    Y,
1755    Z,
1756}
1757
1758#[derive(Clone, Debug, Error)]
1759#[non_exhaustive]
1760pub enum TextureDimensionError {
1761    #[error("Dimension {0:?} is zero")]
1762    Zero(TextureErrorDimension),
1763    #[error("Dimension {dim:?} value {given} exceeds the limit of {limit}")]
1764    LimitExceeded {
1765        dim: TextureErrorDimension,
1766        given: u32,
1767        limit: u32,
1768    },
1769    #[error("Sample count {0} is invalid")]
1770    InvalidSampleCount(u32),
1771    #[error("Width {width} is not a multiple of {format:?}'s block width ({block_width})")]
1772    NotMultipleOfBlockWidth {
1773        width: u32,
1774        block_width: u32,
1775        format: wgt::TextureFormat,
1776    },
1777    #[error("Height {height} is not a multiple of {format:?}'s block height ({block_height})")]
1778    NotMultipleOfBlockHeight {
1779        height: u32,
1780        block_height: u32,
1781        format: wgt::TextureFormat,
1782    },
1783    #[error(
1784        "Width {width} is not a multiple of {format:?}'s width multiple requirement ({multiple})"
1785    )]
1786    WidthNotMultipleOf {
1787        width: u32,
1788        multiple: u32,
1789        format: wgt::TextureFormat,
1790    },
1791    #[error("Height {height} is not a multiple of {format:?}'s height multiple requirement ({multiple})")]
1792    HeightNotMultipleOf {
1793        height: u32,
1794        multiple: u32,
1795        format: wgt::TextureFormat,
1796    },
1797    #[error("Multisampled texture depth or array layers must be 1, got {0}")]
1798    MultisampledDepthOrArrayLayer(u32),
1799}
1800
1801impl WebGpuError for TextureDimensionError {
1802    fn webgpu_error_type(&self) -> ErrorType {
1803        ErrorType::Validation
1804    }
1805}
1806
1807#[derive(Clone, Debug, Error)]
1808#[non_exhaustive]
1809pub enum CreateTextureError {
1810    #[error(transparent)]
1811    Device(#[from] DeviceError),
1812    #[error(transparent)]
1813    CreateTextureView(#[from] CreateTextureViewError),
1814    #[error("Invalid usage flags {0:?}")]
1815    InvalidUsage(wgt::TextureUsages),
1816    #[error(transparent)]
1817    InvalidDimension(#[from] TextureDimensionError),
1818    #[error("Depth texture ({1:?}) can't be created as {0:?}")]
1819    InvalidDepthDimension(wgt::TextureDimension, wgt::TextureFormat),
1820    #[error("Compressed texture ({1:?}) can't be created as {0:?}")]
1821    InvalidCompressedDimension(wgt::TextureDimension, wgt::TextureFormat),
1822    #[error(
1823        "Texture descriptor mip level count {requested} is invalid, maximum allowed is {maximum}"
1824    )]
1825    InvalidMipLevelCount { requested: u32, maximum: u32 },
1826    #[error(
1827        "Texture usages {0:?} are not allowed on a texture of type {1:?}{downlevel_suffix}",
1828        downlevel_suffix = if *.2 { " due to downlevel restrictions" } else { "" }
1829    )]
1830    InvalidFormatUsages(wgt::TextureUsages, wgt::TextureFormat, bool),
1831    #[error("The view format {0:?} is not compatible with texture format {1:?}, only changing srgb-ness is allowed.")]
1832    InvalidViewFormat(wgt::TextureFormat, wgt::TextureFormat),
1833    #[error("Transient texture usage must be equal to `TRANSIENT_ATTACHMENT | RENDER_ATTACHMENT`, but got `{0:?}`")]
1834    InvalidTransientTextureUsage(wgt::TextureUsages),
1835    #[error("Transient texture view formats must be empty")]
1836    InvalidTransientTextureViewFormats,
1837    #[error("Texture usages {0:?} are not allowed on a texture of dimensions {1:?}")]
1838    InvalidDimensionUsages(wgt::TextureUsages, wgt::TextureDimension),
1839    #[error("Texture usage STORAGE_BINDING is not allowed for multisampled textures")]
1840    InvalidMultisampledStorageBinding,
1841    #[error("Format {0:?} does not support multisampling")]
1842    InvalidMultisampledFormat(wgt::TextureFormat),
1843    #[error("Sample count {0} is not supported by format {1:?} on this device. The WebGPU spec guarantees {2:?} samples are supported by this format. With the TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES feature your device supports {3:?}.")]
1844    InvalidSampleCount(u32, wgt::TextureFormat, Vec<u32>, Vec<u32>),
1845    #[error("Multisampled textures must have RENDER_ATTACHMENT usage")]
1846    MultisampledNotRenderAttachment,
1847    #[error("Transient texture mip level count ({0}) must be 1")]
1848    InvalidTransientTextureMipLevelCount(u32),
1849    #[error("Transient texture layer count ({0}) must be 1")]
1850    InvalidTransientTextureLayerCount(u32),
1851    #[error("Texture format {0:?} can't be used due to missing features")]
1852    MissingFeatures(wgt::TextureFormat, #[source] MissingFeatures),
1853    #[error(transparent)]
1854    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
1855}
1856
1857crate::impl_resource_type!(Texture);
1858impl Labeled for Texture {
1859    fn label(&self) -> &str {
1860        &self.desc.label
1861    }
1862}
1863crate::impl_parent_device!(Texture);
1864crate::impl_storage_item!(Texture);
1865crate::impl_trackable!(Texture);
1866
1867impl Borrow<TextureSelector> for Texture {
1868    fn borrow(&self) -> &TextureSelector {
1869        &self.full_range
1870    }
1871}
1872
1873impl WebGpuError for CreateTextureError {
1874    fn webgpu_error_type(&self) -> ErrorType {
1875        match self {
1876            Self::Device(e) => e.webgpu_error_type(),
1877            Self::CreateTextureView(e) => e.webgpu_error_type(),
1878            Self::InvalidDimension(e) => e.webgpu_error_type(),
1879            Self::MissingFeatures(_, e) => e.webgpu_error_type(),
1880            Self::MissingDownlevelFlags(e) => e.webgpu_error_type(),
1881
1882            Self::InvalidUsage(_)
1883            | Self::InvalidDepthDimension(_, _)
1884            | Self::InvalidCompressedDimension(_, _)
1885            | Self::InvalidMipLevelCount { .. }
1886            | Self::InvalidFormatUsages(_, _, _)
1887            | Self::InvalidViewFormat(_, _)
1888            | Self::InvalidDimensionUsages(_, _)
1889            | Self::InvalidMultisampledStorageBinding
1890            | Self::InvalidMultisampledFormat(_)
1891            | Self::InvalidSampleCount(..)
1892            | Self::InvalidTransientTextureUsage(_)
1893            | Self::InvalidTransientTextureMipLevelCount(_)
1894            | Self::InvalidTransientTextureLayerCount(_)
1895            | Self::InvalidTransientTextureViewFormats
1896            | Self::MultisampledNotRenderAttachment => ErrorType::Validation,
1897        }
1898    }
1899}
1900
1901/// Describes a [`TextureView`].
1902#[derive(Clone, Debug, Default, Eq, PartialEq)]
1903#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
1904#[cfg_attr(feature = "serde", serde(default))]
1905pub struct TextureViewDescriptor<'a> {
1906    /// Debug label of the texture view.
1907    ///
1908    /// This will show up in graphics debuggers for easy identification.
1909    pub label: Label<'a>,
1910    /// Format of the texture view, or `None` for the same format as the texture
1911    /// itself.
1912    ///
1913    /// At this time, it must be the same the underlying format of the texture.
1914    pub format: Option<wgt::TextureFormat>,
1915    /// The dimension of the texture view.
1916    ///
1917    /// - For 1D textures, this must be `D1`.
1918    /// - For 2D textures it must be one of `D2`, `D2Array`, `Cube`, or `CubeArray`.
1919    /// - For 3D textures it must be `D3`.
1920    pub dimension: Option<wgt::TextureViewDimension>,
1921    /// The allowed usage(s) for the texture view. Must be a subset of the usage flags of the texture.
1922    /// If not provided, defaults to the full set of usage flags of the texture.
1923    pub usage: Option<wgt::TextureUsages>,
1924    /// Range within the texture that is accessible via this view.
1925    pub range: wgt::ImageSubresourceRange,
1926}
1927
1928#[derive(Debug)]
1929pub(crate) struct HalTextureViewDescriptor {
1930    pub texture_format: wgt::TextureFormat,
1931    pub format: wgt::TextureFormat,
1932    pub usage: wgt::TextureUsages,
1933    pub dimension: wgt::TextureViewDimension,
1934    pub range: wgt::ImageSubresourceRange,
1935}
1936
1937impl HalTextureViewDescriptor {
1938    pub fn aspects(&self) -> hal::FormatAspects {
1939        hal::FormatAspects::new(self.texture_format, self.range.aspect)
1940    }
1941}
1942
1943#[derive(Debug, Copy, Clone, Error)]
1944pub enum TextureViewNotRenderableReason {
1945    #[error("The texture this view references doesn't include the RENDER_ATTACHMENT usage. Provided usages: {0:?}")]
1946    Usage(wgt::TextureUsages),
1947    #[error("The dimension of this texture view is not 2D. View dimension: {0:?}")]
1948    Dimension(wgt::TextureViewDimension),
1949    #[error("This texture view has more than one mipmap level. View mipmap levels: {0:?}")]
1950    MipLevelCount(u32),
1951    #[error("This texture view has more than one array layer. View array layers: {0:?}")]
1952    ArrayLayerCount(u32),
1953    #[error(
1954        "The aspects of this texture view are a subset of the aspects in the original texture. Aspects: {0:?}"
1955    )]
1956    Aspects(hal::FormatAspects),
1957}
1958
1959#[derive(Debug)]
1960pub struct TextureView {
1961    pub(crate) raw: Snatchable<Box<dyn hal::DynTextureView>>,
1962    // if it's a surface texture - it's none
1963    pub(crate) parent: Arc<Texture>,
1964    pub(crate) device: Arc<Device>,
1965    pub(crate) desc: HalTextureViewDescriptor,
1966    pub(crate) format_features: wgt::TextureFormatFeatures,
1967    /// This is `Err` only if the texture view is not renderable
1968    pub(crate) render_extent: Result<wgt::Extent3d, TextureViewNotRenderableReason>,
1969    pub(crate) samples: u32,
1970    pub(crate) selector: TextureSelector,
1971    /// The `label` from the descriptor used to create the resource.
1972    pub(crate) label: String,
1973}
1974
1975impl Drop for TextureView {
1976    fn drop(&mut self) {
1977        if let Some(raw) = self.raw.take() {
1978            resource_log!("Destroy raw {}", self.error_ident());
1979            unsafe {
1980                self.device.raw().destroy_texture_view(raw);
1981            }
1982        }
1983    }
1984}
1985
1986impl RawResourceAccess for TextureView {
1987    type DynResource = dyn hal::DynTextureView;
1988
1989    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
1990        self.raw.get(guard).map(|it| it.as_ref())
1991    }
1992
1993    fn try_raw<'a>(
1994        &'a self,
1995        guard: &'a SnatchGuard,
1996    ) -> Result<&'a Self::DynResource, DestroyedResourceError> {
1997        self.parent.check_destroyed(guard)?;
1998
1999        self.raw(guard)
2000            .ok_or_else(|| DestroyedResourceError(self.error_ident()))
2001    }
2002}
2003
2004impl TextureView {
2005    /// Checks that the given texture usage contains the required texture usage,
2006    /// returns an error otherwise.
2007    pub(crate) fn check_usage(
2008        &self,
2009        expected: wgt::TextureUsages,
2010    ) -> Result<(), MissingTextureUsageError> {
2011        if self.desc.usage.contains(expected) {
2012            Ok(())
2013        } else {
2014            Err(MissingTextureUsageError {
2015                res: self.error_ident(),
2016                actual: self.desc.usage,
2017                expected,
2018            })
2019        }
2020    }
2021}
2022
2023#[derive(Clone, Debug, Error)]
2024#[non_exhaustive]
2025pub enum CreateTextureViewError {
2026    #[error(transparent)]
2027    Device(#[from] DeviceError),
2028    #[error(transparent)]
2029    DestroyedResource(#[from] DestroyedResourceError),
2030    #[error("Invalid texture view dimension `{view:?}` with texture of dimension `{texture:?}`")]
2031    InvalidTextureViewDimension {
2032        view: wgt::TextureViewDimension,
2033        texture: wgt::TextureDimension,
2034    },
2035    #[error("Texture view format `{0:?}` cannot be used as a render attachment. Make sure the format supports RENDER_ATTACHMENT usage and required device features are enabled.")]
2036    TextureViewFormatNotRenderable(wgt::TextureFormat),
2037    #[error("Texture view format `{0:?}` cannot be used as a storage binding. Make sure the format supports STORAGE usage and required device features are enabled.")]
2038    TextureViewFormatNotStorage(wgt::TextureFormat),
2039    #[error("Texture view usages (`{view:?}`) must be a subset of the texture's original usages (`{texture:?}`)")]
2040    InvalidTextureViewUsage {
2041        view: wgt::TextureUsages,
2042        texture: wgt::TextureUsages,
2043    },
2044    #[error("Texture view dimension `{0:?}` cannot be used with a multisampled texture")]
2045    InvalidMultisampledTextureViewDimension(wgt::TextureViewDimension),
2046    #[error(
2047        "TextureView has an arrayLayerCount of {depth}. Views of type `Cube` must have arrayLayerCount of 6."
2048    )]
2049    InvalidCubemapTextureDepth { depth: u32 },
2050    #[error("TextureView has an arrayLayerCount of {depth}. Views of type `CubeArray` must have an arrayLayerCount that is a multiple of 6.")]
2051    InvalidCubemapArrayTextureDepth { depth: u32 },
2052    #[error("Source texture width and height must be equal for a texture view of dimension `Cube`/`CubeArray`")]
2053    InvalidCubeTextureViewSize,
2054    #[error("Mip level count is 0")]
2055    ZeroMipLevelCount,
2056    #[error("Array layer count is 0")]
2057    ZeroArrayLayerCount,
2058    #[error(
2059        "`TextureView` starts at mip level {base_mip_level} and spans {mip_level_count} mip \
2060        levels, but the texture view only has {total} total mip level(s)"
2061    )]
2062    TooManyMipLevels {
2063        base_mip_level: u32,
2064        mip_level_count: u32,
2065        total: u32,
2066    },
2067    #[error(
2068        "`TextureView` starts at array layer {base_array_layer} and spans {array_layer_count}) \
2069        array layers, but the texture view only has {total} total layer(s)"
2070    )]
2071    TooManyArrayLayers {
2072        base_array_layer: u32,
2073        array_layer_count: u32,
2074        total: u32,
2075    },
2076    #[error("Requested array layer count {requested} is not valid for the target view dimension {dim:?}")]
2077    InvalidArrayLayerCount {
2078        requested: u32,
2079        dim: wgt::TextureViewDimension,
2080    },
2081    #[error(
2082        "Aspect {requested_aspect:?} is not a valid aspect of the source texture format {texture_format:?}"
2083    )]
2084    InvalidAspect {
2085        texture_format: wgt::TextureFormat,
2086        requested_aspect: wgt::TextureAspect,
2087    },
2088    #[error(
2089        "Trying to create a view of format {view:?} of a texture with format {texture:?}, \
2090         but this view format is not present in the texture's viewFormat array"
2091    )]
2092    FormatReinterpretation {
2093        texture: wgt::TextureFormat,
2094        view: wgt::TextureFormat,
2095    },
2096    #[error(
2097        "The texture view (`{view:?}`) from transient texture (`{texture:?}`) must have the same usage"
2098    )]
2099    InvalidTransientTextureViewUsage {
2100        texture: wgt::TextureUsages,
2101        view: wgt::TextureUsages,
2102    },
2103    #[error(transparent)]
2104    InvalidResource(#[from] InvalidResourceError),
2105    #[error(transparent)]
2106    MissingFeatures(#[from] MissingFeatures),
2107}
2108
2109impl From<InvalidOrDestroyedResourceError> for CreateTextureViewError {
2110    fn from(value: InvalidOrDestroyedResourceError) -> Self {
2111        match value {
2112            InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
2113            InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
2114        }
2115    }
2116}
2117
2118impl WebGpuError for CreateTextureViewError {
2119    fn webgpu_error_type(&self) -> ErrorType {
2120        match self {
2121            Self::Device(e) => e.webgpu_error_type(),
2122
2123            Self::InvalidTextureViewDimension { .. }
2124            | Self::InvalidResource(_)
2125            | Self::InvalidMultisampledTextureViewDimension(_)
2126            | Self::InvalidCubemapTextureDepth { .. }
2127            | Self::InvalidCubemapArrayTextureDepth { .. }
2128            | Self::InvalidCubeTextureViewSize
2129            | Self::ZeroMipLevelCount
2130            | Self::ZeroArrayLayerCount
2131            | Self::TooManyMipLevels { .. }
2132            | Self::TooManyArrayLayers { .. }
2133            | Self::InvalidArrayLayerCount { .. }
2134            | Self::InvalidAspect { .. }
2135            | Self::FormatReinterpretation { .. }
2136            | Self::DestroyedResource(_)
2137            | Self::TextureViewFormatNotRenderable(_)
2138            | Self::TextureViewFormatNotStorage(_)
2139            | Self::InvalidTextureViewUsage { .. }
2140            | Self::InvalidTransientTextureViewUsage { .. }
2141            | Self::MissingFeatures(_) => ErrorType::Validation,
2142        }
2143    }
2144}
2145
2146crate::impl_resource_type!(TextureView);
2147crate::impl_labeled!(TextureView);
2148crate::impl_parent_device!(TextureView);
2149crate::impl_storage_item!(TextureView);
2150
2151pub type ExternalTextureDescriptor<'a> = wgt::ExternalTextureDescriptor<Label<'a>>;
2152
2153#[derive(Debug)]
2154pub struct ExternalTexture {
2155    pub(crate) device: Arc<Device>,
2156    /// Between 1 and 3 (inclusive) planes of texture data.
2157    pub(crate) planes: arrayvec::ArrayVec<Arc<TextureView>, 3>,
2158    /// Buffer containing a [`crate::device::resource::ExternalTextureParams`]
2159    /// describing the external texture.
2160    pub(crate) params: Arc<Buffer>,
2161    /// The `label` from the descriptor used to create the resource.
2162    pub(crate) label: String,
2163    pub(crate) tracking_data: TrackingData,
2164}
2165
2166impl Drop for ExternalTexture {
2167    fn drop(&mut self) {
2168        resource_log!("Destroy raw {}", self.error_ident());
2169    }
2170}
2171
2172impl ExternalTexture {
2173    pub fn destroy(self: &Arc<Self>) {
2174        self.params.destroy();
2175    }
2176}
2177
2178#[derive(Clone, Debug, Error)]
2179#[non_exhaustive]
2180pub enum CreateExternalTextureError {
2181    #[error(transparent)]
2182    Device(#[from] DeviceError),
2183    #[error(transparent)]
2184    MissingFeatures(#[from] MissingFeatures),
2185    #[error(transparent)]
2186    InvalidResource(#[from] InvalidResourceError),
2187    #[error(transparent)]
2188    CreateBuffer(#[from] CreateBufferError),
2189    #[error(transparent)]
2190    QueueWrite(#[from] queue::QueueWriteError),
2191    #[error("External texture format {format:?} expects {expected} planes, but given {provided}")]
2192    IncorrectPlaneCount {
2193        format: wgt::ExternalTextureFormat,
2194        expected: usize,
2195        provided: usize,
2196    },
2197    #[error("External texture planes cannot be multisampled, but given view with samples = {0}")]
2198    InvalidPlaneMultisample(u32),
2199    #[error("External texture planes expect a filterable float sample type, but given view with format {format:?} (sample type {sample_type:?})")]
2200    InvalidPlaneSampleType {
2201        format: wgt::TextureFormat,
2202        sample_type: wgt::TextureSampleType,
2203    },
2204    #[error("External texture planes expect 2D dimension, but given view with dimension = {0:?}")]
2205    InvalidPlaneDimension(wgt::TextureViewDimension),
2206    #[error(transparent)]
2207    MissingTextureUsage(#[from] MissingTextureUsageError),
2208    #[error("External texture format {format:?} plane {plane} expects format with {expected} components but given view with format {provided:?} ({} components)",
2209        provided.components())]
2210    InvalidPlaneFormat {
2211        format: wgt::ExternalTextureFormat,
2212        plane: usize,
2213        expected: u8,
2214        provided: wgt::TextureFormat,
2215    },
2216}
2217
2218impl WebGpuError for CreateExternalTextureError {
2219    fn webgpu_error_type(&self) -> ErrorType {
2220        match self {
2221            CreateExternalTextureError::Device(e) => e.webgpu_error_type(),
2222            CreateExternalTextureError::MissingFeatures(e) => e.webgpu_error_type(),
2223            CreateExternalTextureError::InvalidResource(e) => e.webgpu_error_type(),
2224            CreateExternalTextureError::CreateBuffer(e) => e.webgpu_error_type(),
2225            CreateExternalTextureError::QueueWrite(e) => e.webgpu_error_type(),
2226            CreateExternalTextureError::MissingTextureUsage(e) => e.webgpu_error_type(),
2227            CreateExternalTextureError::IncorrectPlaneCount { .. }
2228            | CreateExternalTextureError::InvalidPlaneMultisample(_)
2229            | CreateExternalTextureError::InvalidPlaneSampleType { .. }
2230            | CreateExternalTextureError::InvalidPlaneDimension(_)
2231            | CreateExternalTextureError::InvalidPlaneFormat { .. } => ErrorType::Validation,
2232        }
2233    }
2234}
2235
2236crate::impl_resource_type!(ExternalTexture);
2237crate::impl_labeled!(ExternalTexture);
2238crate::impl_parent_device!(ExternalTexture);
2239crate::impl_storage_item!(ExternalTexture);
2240crate::impl_trackable!(ExternalTexture);
2241
2242/// Describes a [`Sampler`]
2243#[derive(Clone, Debug, PartialEq)]
2244#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
2245pub struct SamplerDescriptor<'a> {
2246    /// Debug label of the sampler.
2247    ///
2248    /// This will show up in graphics debuggers for easy identification.
2249    pub label: Label<'a>,
2250    /// How to deal with out of bounds accesses in the u (i.e. x) direction
2251    pub address_modes: [wgt::AddressMode; 3],
2252    /// How to filter the texture when it needs to be magnified (made larger)
2253    pub mag_filter: wgt::FilterMode,
2254    /// How to filter the texture when it needs to be minified (made smaller)
2255    pub min_filter: wgt::FilterMode,
2256    /// How to filter between mip map levels
2257    pub mipmap_filter: wgt::MipmapFilterMode,
2258    /// Minimum level of detail (i.e. mip level) to use
2259    pub lod_min_clamp: f32,
2260    /// Maximum level of detail (i.e. mip level) to use
2261    pub lod_max_clamp: f32,
2262    /// If this is enabled, this is a comparison sampler using the given comparison function.
2263    pub compare: Option<wgt::CompareFunction>,
2264    /// Must be at least 1. If this is not 1, all filter modes must be linear.
2265    pub anisotropy_clamp: u16,
2266    /// Border color to use when address_mode is
2267    /// [`AddressMode::ClampToBorder`](wgt::AddressMode::ClampToBorder)
2268    pub border_color: Option<wgt::SamplerBorderColor>,
2269}
2270
2271#[derive(Debug)]
2272pub struct Sampler {
2273    pub(crate) raw: ManuallyDrop<Box<dyn hal::DynSampler>>,
2274    pub(crate) device: Arc<Device>,
2275    /// The `label` from the descriptor used to create the resource.
2276    pub(crate) label: String,
2277    pub(crate) tracking_data: TrackingData,
2278    /// `true` if this is a comparison sampler
2279    pub(crate) comparison: bool,
2280    /// `true` if this is a filtering sampler
2281    pub(crate) filtering: bool,
2282}
2283
2284impl Drop for Sampler {
2285    fn drop(&mut self) {
2286        resource_log!("Destroy raw {}", self.error_ident());
2287        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
2288        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
2289        unsafe {
2290            self.device.raw().destroy_sampler(raw);
2291        }
2292    }
2293}
2294
2295impl Sampler {
2296    pub(crate) fn raw(&self) -> &dyn hal::DynSampler {
2297        self.raw.as_ref()
2298    }
2299}
2300
2301#[derive(Copy, Clone)]
2302pub enum SamplerFilterErrorType {
2303    MagFilter,
2304    MinFilter,
2305    MipmapFilter,
2306}
2307
2308impl fmt::Debug for SamplerFilterErrorType {
2309    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2310        match *self {
2311            SamplerFilterErrorType::MagFilter => write!(f, "magFilter"),
2312            SamplerFilterErrorType::MinFilter => write!(f, "minFilter"),
2313            SamplerFilterErrorType::MipmapFilter => write!(f, "mipmapFilter"),
2314        }
2315    }
2316}
2317
2318#[derive(Clone, Debug, Error)]
2319#[non_exhaustive]
2320pub enum CreateSamplerError {
2321    #[error(transparent)]
2322    Device(#[from] DeviceError),
2323    #[error("Invalid lodMinClamp: {0}. Must be greater or equal to 0.0")]
2324    InvalidLodMinClamp(f32),
2325    #[error("Invalid lodMaxClamp: {lod_max_clamp}. Must be greater or equal to lodMinClamp (which is {lod_min_clamp}).")]
2326    InvalidLodMaxClamp {
2327        lod_min_clamp: f32,
2328        lod_max_clamp: f32,
2329    },
2330    #[error("Invalid anisotropic clamp: {0}. Must be at least 1.")]
2331    InvalidAnisotropy(u16),
2332    #[error("Invalid filter mode for {filter_type:?}: {filter_mode:?}. When anistropic clamp is not 1 (it is {anisotropic_clamp}), all filter modes must be linear.")]
2333    InvalidFilterModeWithAnisotropy {
2334        filter_type: SamplerFilterErrorType,
2335        filter_mode: wgt::FilterMode,
2336        anisotropic_clamp: u16,
2337    },
2338    #[error("Invalid filter mode for {filter_type:?}: {filter_mode:?}. When anistropic clamp is not 1 (it is {anisotropic_clamp}), all filter modes must be linear.")]
2339    InvalidMipmapFilterModeWithAnisotropy {
2340        filter_type: SamplerFilterErrorType,
2341        filter_mode: wgt::MipmapFilterMode,
2342        anisotropic_clamp: u16,
2343    },
2344    #[error(transparent)]
2345    MissingFeatures(#[from] MissingFeatures),
2346}
2347
2348crate::impl_resource_type!(Sampler);
2349crate::impl_labeled!(Sampler);
2350crate::impl_parent_device!(Sampler);
2351crate::impl_storage_item!(Sampler);
2352crate::impl_trackable!(Sampler);
2353
2354impl WebGpuError for CreateSamplerError {
2355    fn webgpu_error_type(&self) -> ErrorType {
2356        match self {
2357            Self::Device(e) => e.webgpu_error_type(),
2358            Self::MissingFeatures(e) => e.webgpu_error_type(),
2359
2360            Self::InvalidLodMinClamp(_)
2361            | Self::InvalidLodMaxClamp { .. }
2362            | Self::InvalidAnisotropy(_)
2363            | Self::InvalidFilterModeWithAnisotropy { .. }
2364            | Self::InvalidMipmapFilterModeWithAnisotropy { .. } => ErrorType::Validation,
2365        }
2366    }
2367}
2368
2369#[derive(Clone, Debug, Error)]
2370#[non_exhaustive]
2371pub enum CreateQuerySetError {
2372    #[error(transparent)]
2373    Device(#[from] DeviceError),
2374    #[error("QuerySets cannot be made with zero queries")]
2375    ZeroCount,
2376    #[error("{count} is too many queries for a single QuerySet. QuerySets cannot be made more than {maximum} queries.")]
2377    TooManyQueries { count: u32, maximum: u32 },
2378    #[error(transparent)]
2379    MissingFeatures(#[from] MissingFeatures),
2380}
2381
2382impl WebGpuError for CreateQuerySetError {
2383    fn webgpu_error_type(&self) -> ErrorType {
2384        match self {
2385            Self::Device(e) => e.webgpu_error_type(),
2386            Self::MissingFeatures(e) => e.webgpu_error_type(),
2387
2388            Self::TooManyQueries { .. } | Self::ZeroCount => ErrorType::Validation,
2389        }
2390    }
2391}
2392
2393pub type QuerySetDescriptor<'a> = wgt::QuerySetDescriptor<Label<'a>>;
2394
2395#[derive(Debug)]
2396pub struct QuerySet {
2397    pub(crate) raw: Snatchable<Box<dyn hal::DynQuerySet>>,
2398    pub(crate) device: Arc<Device>,
2399    /// The `label` from the descriptor used to create the resource.
2400    pub(crate) label: String,
2401    pub(crate) tracking_data: TrackingData,
2402    pub(crate) desc: wgt::QuerySetDescriptor<()>,
2403    pub(crate) initialized_slots: Mutex<bit_vec::BitVec>,
2404}
2405
2406impl RawResourceAccess for QuerySet {
2407    type DynResource = dyn hal::DynQuerySet;
2408
2409    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2410        self.raw.get(guard).map(|b| b.as_ref())
2411    }
2412}
2413
2414impl QuerySet {
2415    pub fn destroy(self: &Arc<Self>) {
2416        let device = &self.device;
2417
2418        let temp = {
2419            let mut snatch_guard = self.device.snatchable_lock.write();
2420
2421            let raw = match self.raw.snatch(&mut snatch_guard) {
2422                Some(raw) => raw,
2423                None => {
2424                    // Per spec, it is valid to call `destroy` multiple times.
2425                    return;
2426                }
2427            };
2428
2429            drop(snatch_guard);
2430
2431            queue::TempResource::DestroyedQuerySet(DestroyedQuerySet {
2432                raw: ManuallyDrop::new(raw),
2433                device: Arc::clone(&self.device),
2434                label: self.label().to_owned(),
2435            })
2436        };
2437
2438        let Some(queue) = device.get_queue() else {
2439            return;
2440        };
2441
2442        let mut life_lock = queue.lock_life();
2443        let last_submit_index = life_lock.get_query_set_latest_submission_index(self);
2444        if let Some(last_submit_index) = last_submit_index {
2445            life_lock.schedule_resource_destruction(temp, last_submit_index);
2446        }
2447    }
2448}
2449
2450impl Drop for QuerySet {
2451    fn drop(&mut self) {
2452        resource_log!("Destroy raw {}", self.error_ident());
2453        if let Some(raw) = self.raw.take() {
2454            // SAFETY: We are in the Drop impl and we don't use raw anymore after this point.
2455            unsafe {
2456                self.device.raw().destroy_query_set(raw);
2457            }
2458        }
2459    }
2460}
2461
2462crate::impl_resource_type!(QuerySet);
2463crate::impl_labeled!(QuerySet);
2464crate::impl_parent_device!(QuerySet);
2465crate::impl_storage_item!(QuerySet);
2466crate::impl_trackable!(QuerySet);
2467
2468/// A query set that has been marked as destroyed and is staged for actual deletion soon
2469#[derive(Debug)]
2470pub struct DestroyedQuerySet {
2471    raw: ManuallyDrop<Box<dyn hal::DynQuerySet>>,
2472    device: Arc<Device>,
2473    label: String,
2474}
2475
2476impl DestroyedQuerySet {
2477    pub fn label(&self) -> &dyn fmt::Debug {
2478        &self.label
2479    }
2480}
2481
2482impl Drop for DestroyedQuerySet {
2483    fn drop(&mut self) {
2484        resource_log!("Destroy raw QuerySet (destroyed) {:?}", self.label());
2485        // SAFETY: We are in the Drop impl and we don't use self.raw anymore after this point.
2486        let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
2487        unsafe {
2488            hal::DynDevice::destroy_query_set(self.device.raw(), raw);
2489        }
2490    }
2491}
2492
2493pub type BlasDescriptor<'a> = wgt::CreateBlasDescriptor<Label<'a>>;
2494pub type TlasDescriptor<'a> = wgt::CreateTlasDescriptor<Label<'a>>;
2495
2496pub type BlasPrepareCompactResult = Result<(), BlasPrepareCompactError>;
2497
2498#[cfg(send_sync)]
2499pub type BlasCompactCallback = Box<dyn FnOnce(BlasPrepareCompactResult) + Send + 'static>;
2500#[cfg(not(send_sync))]
2501pub type BlasCompactCallback = Box<dyn FnOnce(BlasPrepareCompactResult) + 'static>;
2502
2503pub(crate) struct BlasPendingCompact {
2504    pub(crate) op: Option<BlasCompactCallback>,
2505    // hold the parent alive while the mapping is active
2506    pub(crate) _parent_blas: Arc<Blas>,
2507}
2508
2509impl fmt::Debug for BlasPendingCompact {
2510    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2511        f.debug_struct("BlasPendingCompact")
2512            .field("op", &())
2513            .field("_parent_blas", &self._parent_blas)
2514            .finish()
2515    }
2516}
2517
2518#[derive(Debug)]
2519pub(crate) enum BlasCompactState {
2520    /// Created from a compact operation.
2521    Compacted,
2522    /// Waiting for GPU to be done before mapping to get compacted size
2523    Waiting(BlasPendingCompact),
2524    /// Ready to be compacted
2525    Ready { size: wgt::BufferAddress },
2526    /// Ready to prepare to compact.
2527    Idle,
2528}
2529
2530#[cfg(send_sync)]
2531unsafe impl Send for BlasCompactState {}
2532#[cfg(send_sync)]
2533unsafe impl Sync for BlasCompactState {}
2534
2535#[derive(Debug)]
2536pub struct Blas {
2537    pub(crate) raw: Snatchable<Box<dyn hal::DynAccelerationStructure>>,
2538    pub(crate) device: Arc<Device>,
2539    pub(crate) size_info: hal::AccelerationStructureBuildSizes,
2540    pub(crate) sizes: wgt::BlasGeometrySizeDescriptors,
2541    pub(crate) flags: wgt::AccelerationStructureFlags,
2542    pub(crate) update_mode: wgt::AccelerationStructureUpdateMode,
2543    pub(crate) built_index: RwLock<Option<NonZeroU64>>,
2544    pub(crate) handle: u64,
2545    /// The `label` from the descriptor used to create the resource.
2546    pub(crate) label: String,
2547    pub(crate) tracking_data: TrackingData,
2548    pub(crate) compaction_buffer: Option<ManuallyDrop<Box<dyn hal::DynBuffer>>>,
2549    pub(crate) compacted_state: Mutex<BlasCompactState>,
2550}
2551
2552impl Drop for Blas {
2553    fn drop(&mut self) {
2554        resource_log!("Destroy raw {}", self.error_ident());
2555        // SAFETY: We are in the Drop impl, and we don't use self.raw or self.compaction_buffer anymore after this point.
2556        if let Some(raw) = self.raw.take() {
2557            unsafe {
2558                self.device.raw().destroy_acceleration_structure(raw);
2559            }
2560        }
2561        if let Some(mut raw) = self.compaction_buffer.take() {
2562            unsafe {
2563                self.device
2564                    .raw()
2565                    .destroy_buffer(ManuallyDrop::take(&mut raw))
2566            }
2567        }
2568    }
2569}
2570
2571impl RawResourceAccess for Blas {
2572    type DynResource = dyn hal::DynAccelerationStructure;
2573
2574    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2575        self.raw.get(guard).map(|it| it.as_ref())
2576    }
2577}
2578
2579impl Blas {
2580    pub(crate) fn prepare_compact_async(
2581        self: &Arc<Self>,
2582        op: Option<BlasCompactCallback>,
2583    ) -> Result<SubmissionIndex, (Option<BlasCompactCallback>, BlasPrepareCompactError)> {
2584        let device = &self.device;
2585        if let Err(e) = device.check_is_valid() {
2586            return Err((op, e.into()));
2587        }
2588
2589        if self.built_index.read().is_none() {
2590            return Err((op, BlasPrepareCompactError::NotBuilt));
2591        }
2592
2593        if !self
2594            .flags
2595            .contains(wgt::AccelerationStructureFlags::ALLOW_COMPACTION)
2596        {
2597            return Err((op, BlasPrepareCompactError::CompactionUnsupported));
2598        }
2599
2600        let mut state = self.compacted_state.lock();
2601        *state = match *state {
2602            BlasCompactState::Compacted => {
2603                return Err((op, BlasPrepareCompactError::DoubleCompaction))
2604            }
2605            BlasCompactState::Waiting(_) => {
2606                return Err((op, BlasPrepareCompactError::CompactionPreparingAlready))
2607            }
2608            BlasCompactState::Ready { .. } => {
2609                return Err((op, BlasPrepareCompactError::CompactionPreparingAlready))
2610            }
2611            BlasCompactState::Idle => BlasCompactState::Waiting(BlasPendingCompact {
2612                op,
2613                _parent_blas: self.clone(),
2614            }),
2615        };
2616
2617        let submit_index = if let Some(queue) = device.get_queue() {
2618            queue.lock_life().prepare_compact(self).unwrap_or(0) // '0' means no wait is necessary
2619        } else {
2620            // We can safely unwrap below since we just set the `compacted_state` to `BlasCompactState::Waiting`.
2621            let (mut callback, status) = self.read_back_compact_size().unwrap();
2622            if let Some(callback) = callback.take() {
2623                callback(status);
2624            }
2625            0
2626        };
2627
2628        Ok(submit_index)
2629    }
2630
2631    /// This function returns [`None`] only if [`Self::compacted_state`] is not [`BlasCompactState::Waiting`].
2632    #[must_use]
2633    pub(crate) fn read_back_compact_size(&self) -> Option<BlasCompactReadyPendingClosure> {
2634        let mut state = self.compacted_state.lock();
2635        let pending_compact = match mem::replace(&mut *state, BlasCompactState::Idle) {
2636            BlasCompactState::Waiting(pending_mapping) => pending_mapping,
2637            // Compaction cancelled e.g. by rebuild
2638            BlasCompactState::Idle => return None,
2639            BlasCompactState::Ready { .. } => {
2640                unreachable!("This should be validated out by `prepare_for_compaction`")
2641            }
2642            _ => panic!("No pending mapping."),
2643        };
2644        let status = {
2645            let compaction_buffer = self.compaction_buffer.as_ref().unwrap().as_ref();
2646            unsafe {
2647                let map_res = self.device.raw().map_buffer(
2648                    compaction_buffer,
2649                    0..size_of::<wgpu_types::BufferAddress>() as wgt::BufferAddress,
2650                );
2651                match map_res {
2652                    Ok(mapping) => {
2653                        if !mapping.is_coherent {
2654                            #[expect(clippy::single_range_in_vec_init, reason = "intentional")]
2655                            self.device.raw().invalidate_mapped_ranges(
2656                                compaction_buffer,
2657                                &[0..size_of::<wgpu_types::BufferAddress>() as wgt::BufferAddress],
2658                            );
2659                        }
2660                        let size = core::ptr::read_unaligned(
2661                            mapping.ptr.as_ptr().cast::<wgt::BufferAddress>(),
2662                        );
2663                        self.device.raw().unmap_buffer(compaction_buffer);
2664                        if self.size_info.acceleration_structure_size != 0 {
2665                            debug_assert_ne!(size, 0);
2666                        }
2667                        *state = BlasCompactState::Ready { size };
2668                        Ok(())
2669                    }
2670                    Err(err) => Err(BlasPrepareCompactError::from(DeviceError::from_hal(err))),
2671                }
2672            }
2673        };
2674        Some((pending_compact.op, status))
2675    }
2676}
2677
2678crate::impl_resource_type!(Blas);
2679crate::impl_labeled!(Blas);
2680crate::impl_parent_device!(Blas);
2681crate::impl_storage_item!(Blas);
2682crate::impl_trackable!(Blas);
2683
2684#[derive(Debug)]
2685pub struct Tlas {
2686    pub(crate) raw: Snatchable<Box<dyn hal::DynAccelerationStructure>>,
2687    pub(crate) device: Arc<Device>,
2688    pub(crate) size_info: hal::AccelerationStructureBuildSizes,
2689    pub(crate) max_instance_count: u32,
2690    pub(crate) flags: wgt::AccelerationStructureFlags,
2691    pub(crate) update_mode: wgt::AccelerationStructureUpdateMode,
2692    pub(crate) built_index: RwLock<Option<NonZeroU64>>,
2693    pub(crate) dependencies: RwLock<Vec<Arc<Blas>>>,
2694    pub(crate) instance_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
2695    /// The `label` from the descriptor used to create the resource.
2696    pub(crate) label: String,
2697    pub(crate) tracking_data: TrackingData,
2698}
2699
2700impl Drop for Tlas {
2701    fn drop(&mut self) {
2702        unsafe {
2703            resource_log!("Destroy raw {}", self.error_ident());
2704            if let Some(structure) = self.raw.take() {
2705                self.device.raw().destroy_acceleration_structure(structure);
2706            }
2707            let buffer = ManuallyDrop::take(&mut self.instance_buffer);
2708            self.device.raw().destroy_buffer(buffer);
2709        }
2710    }
2711}
2712
2713impl RawResourceAccess for Tlas {
2714    type DynResource = dyn hal::DynAccelerationStructure;
2715
2716    fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2717        self.raw.get(guard).map(|raw| raw.as_ref())
2718    }
2719}
2720
2721crate::impl_resource_type!(Tlas);
2722crate::impl_labeled!(Tlas);
2723crate::impl_parent_device!(Tlas);
2724crate::impl_storage_item!(Tlas);
2725crate::impl_trackable!(Tlas);