1use alloc::{borrow::Cow, borrow::ToOwned as _, boxed::Box, string::String, sync::Arc, vec::Vec};
2use core::{
3 borrow::Borrow,
4 fmt,
5 mem::{self, size_of, ManuallyDrop},
6 num::NonZeroU64,
7 ops::Range,
8 ptr::NonNull,
9};
10use smallvec::SmallVec;
11use thiserror::Error;
12use wgt::{
13 error::{ErrorType, WebGpuError},
14 TextureSelector,
15};
16
17#[cfg(feature = "trace")]
18use crate::device::trace;
19use crate::{
20 binding_model::{BindGroup, BindingError},
21 device::{
22 queue, resource::DeferredDestroy, BufferMapPendingClosure, Device, DeviceError,
23 DeviceMismatch, HostMap, MissingDownlevelFlags, MissingFeatures,
24 },
25 hal_label,
26 init_tracker::{BufferInitTracker, TextureInitTracker},
27 lock::{rank, Mutex, RwLock},
28 ray_tracing::{BlasCompactReadyPendingClosure, BlasPrepareCompactError},
29 resource_log,
30 snatch::{SnatchGuard, Snatchable, Snatchable2},
31 timestamp_normalization::TimestampNormalizationBindGroup,
32 track::{SharedTrackerIndexAllocator, TrackerIndex},
33 weak_vec::WeakVec,
34 Label, LabelHelpers, SubmissionIndex,
35};
36
37#[derive(Debug)]
57pub(crate) struct TrackingData {
58 tracker_index: TrackerIndex,
59 tracker_indices: Arc<SharedTrackerIndexAllocator>,
60}
61
62impl Drop for TrackingData {
63 fn drop(&mut self) {
64 self.tracker_indices.free(self.tracker_index);
65 }
66}
67
68impl TrackingData {
69 pub(crate) fn new(tracker_indices: Arc<SharedTrackerIndexAllocator>) -> Self {
70 Self {
71 tracker_index: tracker_indices.alloc(),
72 tracker_indices,
73 }
74 }
75
76 pub(crate) fn tracker_index(&self) -> TrackerIndex {
77 self.tracker_index
78 }
79}
80
81#[derive(Clone, Debug)]
82#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
83pub struct ResourceErrorIdent {
84 r#type: Cow<'static, str>,
85 label: String,
86}
87
88impl fmt::Display for ResourceErrorIdent {
89 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
90 write!(f, "{} with '{}' label", self.r#type, self.label)
91 }
92}
93
94#[derive(Debug)]
95pub enum ResourceState<T> {
96 Valid(T),
97 Invalid,
98}
99
100impl<T> ResourceState<T> {
101 pub fn as_ref(&self) -> ResourceState<&T> {
102 match self {
103 ResourceState::Valid(v) => ResourceState::Valid(v),
104 ResourceState::Invalid => ResourceState::Invalid,
105 }
106 }
107
108 pub fn valid(self) -> Option<T> {
109 match self {
110 ResourceState::Valid(v) => Some(v),
111 ResourceState::Invalid => None,
112 }
113 }
114}
115
116pub enum DestructibleResourceState<T> {
117 Valid(T),
118 Invalid,
119 Destroyed,
120}
121
122impl<T> DestructibleResourceState<T> {
123 pub fn as_ref(&self) -> DestructibleResourceState<&T> {
124 match self {
125 DestructibleResourceState::Valid(v) => DestructibleResourceState::Valid(v),
126 DestructibleResourceState::Invalid => DestructibleResourceState::Invalid,
127 DestructibleResourceState::Destroyed => DestructibleResourceState::Destroyed,
128 }
129 }
130
131 pub fn take(&mut self) -> DestructibleResourceState<T> {
132 mem::replace(self, DestructibleResourceState::Destroyed)
133 }
134}
135
136#[derive(thiserror::Error, Debug)]
137pub enum InvalidOrDestroyedResourceError {
138 #[error(transparent)]
139 InvalidResource(#[from] InvalidResourceError),
140 #[error(transparent)]
141 DestroyedResource(#[from] DestroyedResourceError),
142}
143
144impl<T> DestructibleResourceState<T> {
145 pub fn maybe_valid(self) -> Option<T> {
146 match self {
147 DestructibleResourceState::Valid(t) => Some(t),
148 DestructibleResourceState::Invalid => None,
149 DestructibleResourceState::Destroyed => None,
150 }
151 }
152}
153
154pub trait ParentDevice: Labeled {
155 fn device(&self) -> &Arc<Device>;
156
157 fn is_equal(self: &Arc<Self>, other: &Arc<Self>) -> bool {
158 Arc::ptr_eq(self, other)
159 }
160
161 fn same_device_as<O: ParentDevice>(&self, other: &O) -> Result<(), DeviceError> {
162 if Arc::ptr_eq(self.device(), other.device()) {
163 Ok(())
164 } else {
165 Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch {
166 res: self.error_ident(),
167 res_device: self.device().error_ident(),
168 target: Some(other.error_ident()),
169 target_device: other.device().error_ident(),
170 })))
171 }
172 }
173
174 fn same_device(&self, device: &Device) -> Result<(), DeviceError> {
175 if core::ptr::eq(&**self.device(), device) {
176 Ok(())
177 } else {
178 Err(DeviceError::DeviceMismatch(Box::new(DeviceMismatch {
179 res: self.error_ident(),
180 res_device: self.device().error_ident(),
181 target: None,
182 target_device: device.error_ident(),
183 })))
184 }
185 }
186}
187
188#[macro_export]
189macro_rules! impl_parent_device {
190 ($ty:ident) => {
191 impl $crate::resource::ParentDevice for $ty {
192 fn device(&self) -> &Arc<Device> {
193 &self.device
194 }
195 }
196 };
197}
198
199pub trait RawResourceAccess: ParentDevice {
201 type DynResource: hal::DynResource + ?Sized;
202
203 fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource>;
208
209 fn try_raw<'a>(
214 &'a self,
215 guard: &'a SnatchGuard,
216 ) -> Result<&'a Self::DynResource, DestroyedResourceError> {
217 self.raw(guard)
218 .ok_or_else(|| DestroyedResourceError(self.error_ident()))
219 }
220}
221
222pub trait ResourceType {
223 const TYPE: &'static str;
224}
225
226#[macro_export]
227macro_rules! impl_resource_type {
228 ($ty:ident) => {
229 impl $crate::resource::ResourceType for $ty {
230 const TYPE: &'static str = stringify!($ty);
231 }
232 };
233}
234
235pub trait Labeled: ResourceType {
236 fn label(&self) -> &str;
242
243 fn error_ident(&self) -> ResourceErrorIdent {
244 ResourceErrorIdent {
245 r#type: Cow::Borrowed(Self::TYPE),
246 label: self.label().to_owned(),
247 }
248 }
249}
250
251#[macro_export]
252macro_rules! impl_labeled {
253 ($ty:ident) => {
254 impl $crate::resource::Labeled for $ty {
255 fn label(&self) -> &str {
256 &self.label
257 }
258 }
259 };
260}
261
262pub(crate) trait Trackable {
263 fn tracker_index(&self) -> TrackerIndex;
264}
265
266#[macro_export]
267macro_rules! impl_trackable {
268 ($ty:ident) => {
269 impl $crate::resource::Trackable for $ty {
270 fn tracker_index(&self) -> $crate::track::TrackerIndex {
271 self.tracking_data.tracker_index()
272 }
273 }
274 };
275}
276
277#[derive(Debug)]
278pub(crate) enum BufferMapState {
279 Init { staging_buffer: StagingBuffer },
281 Waiting(BufferPendingMapping),
283 Active {
285 mapping: hal::BufferMapping,
286 range: hal::MemoryRange,
287 host: HostMap,
288 },
289 Idle,
291}
292
293#[cfg(send_sync)]
294unsafe impl Send for BufferMapState {}
295#[cfg(send_sync)]
296unsafe impl Sync for BufferMapState {}
297
298#[cfg(send_sync)]
299pub type BufferMapCallback = Box<dyn FnOnce(BufferAccessResult) + Send + 'static>;
300#[cfg(not(send_sync))]
301pub type BufferMapCallback = Box<dyn FnOnce(BufferAccessResult) + 'static>;
302
303pub struct BufferMapOperation {
304 pub host: HostMap,
305 pub callback: Option<BufferMapCallback>,
306}
307
308impl fmt::Debug for BufferMapOperation {
309 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
310 f.debug_struct("BufferMapOperation")
311 .field("host", &self.host)
312 .field("callback", &self.callback.as_ref().map(|_| "?"))
313 .finish()
314 }
315}
316
317#[derive(Clone, Debug, Error)]
318#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
319#[non_exhaustive]
320pub enum BufferAccessError {
321 #[error(transparent)]
322 Device(#[from] DeviceError),
323 #[error("Buffer map failed")]
324 Failed,
325 #[error(transparent)]
326 DestroyedResource(#[from] DestroyedResourceError),
327 #[error("Buffer is already mapped")]
328 AlreadyMapped,
329 #[error("Buffer map is pending")]
330 MapAlreadyPending,
331 #[error(transparent)]
332 MissingBufferUsage(#[from] MissingBufferUsageError),
333 #[error("Buffer is not mapped")]
334 NotMapped,
335 #[error(
336 "Buffer map range must start aligned to `MAP_ALIGNMENT` and end to `COPY_BUFFER_ALIGNMENT`"
337 )]
338 UnalignedRange,
339 #[error("Buffer offset invalid: offset {offset} must be multiple of 8")]
340 UnalignedOffset { offset: wgt::BufferAddress },
341 #[error("Buffer range size invalid: range_size {range_size} must be multiple of 4")]
342 UnalignedRangeSize { range_size: wgt::BufferAddress },
343 #[error("Buffer access out of bounds: index {index} would underrun the buffer (limit: {min})")]
344 OutOfBoundsStartOffsetUnderrun {
345 index: wgt::BufferAddress,
346 min: wgt::BufferAddress,
347 },
348 #[error(
349 "Buffer access out of bounds: start offset {index} would overrun the buffer (limit: {max})"
350 )]
351 OutOfBoundsStartOffsetOverrun {
352 index: wgt::BufferAddress,
353 max: wgt::BufferAddress,
354 },
355 #[error(
356 "Buffer access out of bounds: start offset {index} + size {size} would overrun the buffer (limit: {max})"
357 )]
358 OutOfBoundsEndOffsetOverrun {
359 index: wgt::BufferAddress,
360 size: wgt::BufferAddress,
361 max: wgt::BufferAddress,
362 },
363 #[error("Buffer map aborted")]
364 MapAborted,
365 #[error(transparent)]
366 InvalidResource(#[from] InvalidResourceError),
367 #[error("Map start offset ({offset}) is out-of-bounds for buffer of size {buffer_size}")]
368 MapStartOffsetOverrun {
369 offset: wgt::BufferAddress,
370 buffer_size: wgt::BufferAddress,
371 },
372 #[error(
373 "Map end offset (start at {} + size of {}) is out-of-bounds for buffer of size {}",
374 offset,
375 size,
376 buffer_size
377 )]
378 MapEndOffsetOverrun {
379 offset: wgt::BufferAddress,
380 size: wgt::BufferAddress,
381 buffer_size: wgt::BufferAddress,
382 },
383}
384
385impl WebGpuError for BufferAccessError {
386 fn webgpu_error_type(&self) -> ErrorType {
387 match self {
388 Self::Device(e) => e.webgpu_error_type(),
389 Self::InvalidResource(e) => e.webgpu_error_type(),
390 Self::DestroyedResource(e) => e.webgpu_error_type(),
391
392 Self::Failed
393 | Self::AlreadyMapped
394 | Self::MapAlreadyPending
395 | Self::MissingBufferUsage(_)
396 | Self::NotMapped
397 | Self::UnalignedRange
398 | Self::UnalignedOffset { .. }
399 | Self::UnalignedRangeSize { .. }
400 | Self::OutOfBoundsStartOffsetUnderrun { .. }
401 | Self::OutOfBoundsStartOffsetOverrun { .. }
402 | Self::OutOfBoundsEndOffsetOverrun { .. }
403 | Self::MapAborted
404 | Self::MapStartOffsetOverrun { .. }
405 | Self::MapEndOffsetOverrun { .. } => ErrorType::Validation,
406 }
407 }
408}
409
410#[derive(Clone, Debug, Error)]
411#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
412#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")]
413pub struct MissingBufferUsageError {
414 pub(crate) res: ResourceErrorIdent,
415 pub(crate) actual: wgt::BufferUsages,
416 pub(crate) expected: wgt::BufferUsages,
417}
418
419impl WebGpuError for MissingBufferUsageError {
420 fn webgpu_error_type(&self) -> ErrorType {
421 ErrorType::Validation
422 }
423}
424
425#[derive(Clone, Debug, Error)]
426#[error("Usage flags {actual:?} of {res} do not contain required usage flags {expected:?}")]
427pub struct MissingTextureUsageError {
428 pub(crate) res: ResourceErrorIdent,
429 pub(crate) actual: wgt::TextureUsages,
430 pub(crate) expected: wgt::TextureUsages,
431}
432
433impl WebGpuError for MissingTextureUsageError {
434 fn webgpu_error_type(&self) -> ErrorType {
435 ErrorType::Validation
436 }
437}
438
439#[derive(Clone, Debug, Error)]
440#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
441#[error("{0} has been destroyed")]
442pub struct DestroyedResourceError(pub ResourceErrorIdent);
443
444impl WebGpuError for DestroyedResourceError {
445 fn webgpu_error_type(&self) -> ErrorType {
446 ErrorType::Validation
447 }
448}
449
450#[derive(Clone, Debug, Error)]
451#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
452#[error("{0} is invalid")]
453pub struct InvalidResourceError(pub ResourceErrorIdent);
454
455impl WebGpuError for InvalidResourceError {
456 fn webgpu_error_type(&self) -> ErrorType {
457 ErrorType::Validation
458 }
459}
460
461pub enum Fallible<T: ParentDevice> {
462 Valid(Arc<T>),
463 Invalid(Arc<String>),
464}
465
466impl<T: ParentDevice> Fallible<T> {
467 pub fn get(self) -> Result<Arc<T>, InvalidResourceError> {
468 match self {
469 Fallible::Valid(v) => Ok(v),
470 Fallible::Invalid(label) => Err(InvalidResourceError(ResourceErrorIdent {
471 r#type: Cow::Borrowed(T::TYPE),
472 label: (*label).clone(),
473 })),
474 }
475 }
476}
477
478impl<T: ParentDevice> Clone for Fallible<T> {
479 fn clone(&self) -> Self {
480 match self {
481 Self::Valid(v) => Self::Valid(v.clone()),
482 Self::Invalid(l) => Self::Invalid(l.clone()),
483 }
484 }
485}
486
487impl<T: ParentDevice> ResourceType for Fallible<T> {
488 const TYPE: &'static str = T::TYPE;
489}
490
491impl<T: ParentDevice + crate::storage::StorageItem> crate::storage::StorageItem for Fallible<T> {
492 type Marker = T::Marker;
493}
494
495pub type BufferAccessResult = Result<(), BufferAccessError>;
496
497#[derive(Debug)]
498pub(crate) struct BufferPendingMapping {
499 pub(crate) range: Range<wgt::BufferAddress>,
500 pub(crate) op: BufferMapOperation,
501 pub(crate) _parent_buffer: Arc<Buffer>,
503}
504
505pub type BufferDescriptor<'a> = wgt::BufferDescriptor<Label<'a>>;
506
507#[derive(Debug)]
508pub struct Buffer {
509 pub(crate) raw: Snatchable<Box<dyn hal::DynBuffer>>,
510 pub(crate) device: Arc<Device>,
511 pub(crate) usage: wgt::BufferUsages,
512 pub(crate) size: wgt::BufferAddress,
513 pub(crate) initialization_status: RwLock<BufferInitTracker>,
514 pub(crate) label: String,
516 pub(crate) tracking_data: TrackingData,
517 pub(crate) map_state: Mutex<BufferMapState>,
518 pub(crate) destroyed_while_mapped: crate::OnceCellOrLock<DestroyedBuffer>,
521 pub(crate) bind_groups: Mutex<WeakVec<BindGroup>>,
523 pub(crate) timestamp_normalization_bind_group: Snatchable<TimestampNormalizationBindGroup>,
524 pub(crate) indirect_validation_bind_groups: Snatchable<crate::indirect_validation::BindGroups>,
525}
526
527impl Drop for Buffer {
528 fn drop(&mut self) {
529 if let Some(raw) = self.timestamp_normalization_bind_group.take() {
530 raw.dispose(self.device.raw());
531 }
532
533 if let Some(raw) = self.indirect_validation_bind_groups.take() {
534 raw.dispose(self.device.raw());
535 }
536
537 if let Some(raw) = self.raw.take() {
538 resource_log!("Destroy raw {}", self.error_ident());
539 unsafe {
540 self.device.raw().destroy_buffer(raw);
541 }
542 }
543 }
544}
545
546impl RawResourceAccess for Buffer {
547 type DynResource = dyn hal::DynBuffer;
548
549 fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
550 self.raw.get(guard).map(|b| b.as_ref())
551 }
552}
553
554impl Buffer {
555 pub(crate) fn check_destroyed(
556 &self,
557 guard: &SnatchGuard,
558 ) -> Result<(), DestroyedResourceError> {
559 self.raw
560 .get(guard)
561 .map(|_| ())
562 .ok_or_else(|| DestroyedResourceError(self.error_ident()))
563 }
564
565 pub(crate) fn check_usage(
568 &self,
569 expected: wgt::BufferUsages,
570 ) -> Result<(), MissingBufferUsageError> {
571 if self.usage.contains(expected) {
572 Ok(())
573 } else {
574 Err(MissingBufferUsageError {
575 res: self.error_ident(),
576 actual: self.usage,
577 expected,
578 })
579 }
580 }
581
582 pub fn resolve_binding_size(
594 &self,
595 offset: wgt::BufferAddress,
596 binding_size: Option<wgt::BufferSize>,
597 ) -> Result<u64, BindingError> {
598 let buffer_size = self.size;
599
600 match binding_size {
601 Some(binding_size) => match offset.checked_add(binding_size.get()) {
602 Some(end) if end <= buffer_size => Ok(binding_size.get()),
603 _ => Err(BindingError::BindingRangeTooLarge {
604 buffer: self.error_ident(),
605 offset,
606 binding_size: binding_size.get(),
607 buffer_size,
608 }),
609 },
610 None => {
611 buffer_size
612 .checked_sub(offset)
613 .ok_or_else(|| BindingError::BindingOffsetTooLarge {
614 buffer: self.error_ident(),
615 offset,
616 buffer_size,
617 })
618 }
619 }
620 }
621
622 pub fn binding<'a>(
643 &'a self,
644 offset: wgt::BufferAddress,
645 binding_size: Option<wgt::BufferSize>,
646 snatch_guard: &'a SnatchGuard,
647 ) -> Result<(hal::BufferBinding<'a, dyn hal::DynBuffer>, u64), BindingError> {
648 let buf_raw = self.try_raw(snatch_guard)?;
649 let resolved_size = self.resolve_binding_size(offset, binding_size)?;
650 Ok((
653 hal::BufferBinding::new_unchecked(buf_raw, offset, binding_size),
654 resolved_size,
655 ))
656 }
657
658 pub fn map_async(
662 self: &Arc<Self>,
663 offset: wgt::BufferAddress,
664 size: Option<wgt::BufferAddress>,
665 op: BufferMapOperation,
666 ) -> Result<SubmissionIndex, BufferAccessError> {
667 self.try_map_async(offset, size, op)
668 .map_err(|(mut operation, err)| {
669 if let Some(callback) = operation.callback.take() {
670 callback(Err(err.clone()));
671 }
672 err
673 })
674 }
675
676 fn try_map_async(
699 self: &Arc<Self>,
700 offset: wgt::BufferAddress,
701 size: Option<wgt::BufferAddress>,
702 op: BufferMapOperation,
703 ) -> Result<SubmissionIndex, (BufferMapOperation, BufferAccessError)> {
704 let range_size = if let Some(size) = size {
705 size
706 } else {
707 self.size.saturating_sub(offset)
708 };
709
710 if !offset.is_multiple_of(wgt::MAP_ALIGNMENT) {
711 return Err((op, BufferAccessError::UnalignedOffset { offset }));
712 }
713 if !range_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
714 return Err((op, BufferAccessError::UnalignedRangeSize { range_size }));
715 }
716
717 if offset > self.size {
718 return Err((
719 op,
720 BufferAccessError::MapStartOffsetOverrun {
721 offset,
722 buffer_size: self.size,
723 },
724 ));
725 }
726 if range_size > self.size - offset {
728 return Err((
729 op,
730 BufferAccessError::MapEndOffsetOverrun {
731 offset,
732 size: range_size,
733 buffer_size: self.size,
734 },
735 ));
736 }
737 let end_offset = offset + range_size;
738
739 if !offset.is_multiple_of(wgt::MAP_ALIGNMENT)
740 || !end_offset.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT)
741 {
742 return Err((op, BufferAccessError::UnalignedRange));
743 }
744
745 let (pub_usage, internal_use) = match op.host {
746 HostMap::Read => (wgt::BufferUsages::MAP_READ, wgt::BufferUses::MAP_READ),
747 HostMap::Write => (wgt::BufferUsages::MAP_WRITE, wgt::BufferUses::MAP_WRITE),
748 };
749
750 if let Err(e) = self.check_usage(pub_usage) {
751 return Err((op, e.into()));
752 }
753
754 let device = &self.device;
755 if let Err(e) = device.check_is_valid() {
756 return Err((op, e.into()));
757 }
758
759 let submit_index = {
760 let snatch_guard = device.snatchable_lock.read();
761 if let Err(e) = self.check_destroyed(&snatch_guard) {
762 return Err((op, e.into()));
763 }
764
765 {
766 let map_state = &mut *self.map_state.lock();
767 *map_state = match *map_state {
768 BufferMapState::Init { .. } | BufferMapState::Active { .. } => {
769 return Err((op, BufferAccessError::AlreadyMapped));
770 }
771 BufferMapState::Waiting(_) => {
772 return Err((op, BufferAccessError::MapAlreadyPending));
773 }
774 BufferMapState::Idle => BufferMapState::Waiting(BufferPendingMapping {
775 range: offset..end_offset,
776 op,
777 _parent_buffer: self.clone(),
778 }),
779 };
780 }
781
782 if let Some(queue) = device.get_queue().as_ref() {
783 match queue.flush_writes_for_buffer(self, snatch_guard) {
784 Err(err) => {
785 let state = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle);
786 let BufferMapState::Waiting(BufferPendingMapping { op, .. }) = state else {
787 unreachable!();
788 };
789 return Err((op, err));
790 }
791 Ok(()) => {
792 Some(queue.lock_life().map(self).unwrap_or(0))
801 }
802 }
803 } else {
804 None
805 }
806 };
807
808 device
816 .trackers
817 .lock()
818 .buffers
819 .set_single(self, internal_use);
820
821 if let Some(index) = submit_index {
822 Ok(index)
823 } else {
824 let (mut operation, status) = self.map(&device.snatchable_lock.read()).unwrap();
827 if let Some(callback) = operation.callback.take() {
828 callback(status);
829 }
830 Ok(0)
831 }
832 }
833
834 pub fn get_mapped_range(
835 self: &Arc<Self>,
836 offset: wgt::BufferAddress,
837 size: Option<wgt::BufferAddress>,
838 ) -> Result<(NonNull<u8>, u64), BufferAccessError> {
839 {
840 let snatch_guard = self.device.snatchable_lock.read();
841 self.check_destroyed(&snatch_guard)?;
842 }
843
844 let range_size = if let Some(size) = size {
845 size
846 } else {
847 self.size.saturating_sub(offset)
848 };
849
850 if !offset.is_multiple_of(wgt::MAP_ALIGNMENT) {
851 return Err(BufferAccessError::UnalignedOffset { offset });
852 }
853 if !range_size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
854 return Err(BufferAccessError::UnalignedRangeSize { range_size });
855 }
856 let map_state = &*self.map_state.lock();
857 match *map_state {
858 BufferMapState::Init { ref staging_buffer } => {
859 if offset > self.size {
860 return Err(BufferAccessError::MapStartOffsetOverrun {
861 offset,
862 buffer_size: self.size,
863 });
864 }
865 if range_size > self.size - offset {
867 return Err(BufferAccessError::MapEndOffsetOverrun {
868 offset,
869 size: range_size,
870 buffer_size: self.size,
871 });
872 }
873 let ptr = unsafe { staging_buffer.ptr() };
874 let ptr = unsafe { NonNull::new_unchecked(ptr.as_ptr().offset(offset as isize)) };
875 Ok((ptr, range_size))
876 }
877 BufferMapState::Active {
878 ref mapping,
879 ref range,
880 ..
881 } => {
882 if offset > range.end {
883 return Err(BufferAccessError::OutOfBoundsStartOffsetOverrun {
884 index: offset,
885 max: range.end,
886 });
887 }
888 if offset < range.start {
889 return Err(BufferAccessError::OutOfBoundsStartOffsetUnderrun {
890 index: offset,
891 min: range.start,
892 });
893 }
894 if range_size > range.end - offset {
895 return Err(BufferAccessError::OutOfBoundsEndOffsetOverrun {
896 index: offset,
897 size: range_size,
898 max: range.end,
899 });
900 }
901 let relative_offset = (offset - range.start) as isize;
904 unsafe {
905 Ok((
906 NonNull::new_unchecked(mapping.ptr.as_ptr().offset(relative_offset)),
907 range_size,
908 ))
909 }
910 }
911 BufferMapState::Idle | BufferMapState::Waiting(_) => Err(BufferAccessError::NotMapped),
912 }
913 }
914 #[must_use]
917 pub(crate) fn map(&self, snatch_guard: &SnatchGuard) -> Option<BufferMapPendingClosure> {
918 let mapping = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle);
922 let pending_mapping = match mapping {
923 BufferMapState::Waiting(pending_mapping) => pending_mapping,
924 BufferMapState::Idle => return None,
926 BufferMapState::Active { .. } => {
929 *self.map_state.lock() = mapping;
930 return None;
931 }
932 _ => panic!("No pending mapping."),
933 };
934 let status = if pending_mapping.range.start != pending_mapping.range.end {
935 let host = pending_mapping.op.host;
936 let size = pending_mapping.range.end - pending_mapping.range.start;
937 match crate::device::map_buffer(
938 self,
939 pending_mapping.range.start,
940 size,
941 host,
942 snatch_guard,
943 ) {
944 Ok(mapping) => {
945 *self.map_state.lock() = BufferMapState::Active {
946 mapping,
947 range: pending_mapping.range.clone(),
948 host,
949 };
950 Ok(())
951 }
952 Err(e) => Err(e),
953 }
954 } else {
955 *self.map_state.lock() = BufferMapState::Active {
956 mapping: hal::BufferMapping {
957 ptr: NonNull::dangling(),
958 is_coherent: true,
959 },
960 range: pending_mapping.range,
961 host: pending_mapping.op.host,
962 };
963 Ok(())
964 };
965 Some((pending_mapping.op, status))
966 }
967
968 pub fn unmap(self: &Arc<Self>) -> Result<(), BufferAccessError> {
970 if let Some((mut operation, status)) = self.unmap_inner()? {
971 if let Some(callback) = operation.callback.take() {
972 callback(status);
973 }
974 }
975
976 Ok(())
977 }
978
979 fn unmap_inner(self: &Arc<Self>) -> Result<Option<BufferMapPendingClosure>, BufferAccessError> {
980 let device = &self.device;
981 let snatch_guard = device.snatchable_lock.read();
982 let raw_buf = self.try_raw(&snatch_guard)?;
983 let map_state = mem::replace(&mut *self.map_state.lock(), BufferMapState::Idle);
984 match map_state {
985 BufferMapState::Init { staging_buffer } => {
986 #[cfg(feature = "trace")]
987 if let Some(ref mut trace) = *device.trace.lock() {
988 use crate::device::trace::{DataKind, IntoTrace};
989
990 let data = trace.make_binary(DataKind::Bin, staging_buffer.get_data());
991 trace.add(trace::Action::WriteBuffer {
992 id: self.to_trace(),
993 data,
994 offset: 0,
996 size: self.size,
997 queued: true,
998 });
999 }
1000
1001 let staging_buffer = staging_buffer.flush();
1002
1003 if let Some(queue) = device.get_queue() {
1004 let region = wgt::BufferSize::new(self.size).map(|size| hal::BufferCopy {
1005 src_offset: 0,
1006 dst_offset: 0,
1007 size,
1008 });
1009 let transition_src = hal::BufferBarrier {
1010 buffer: staging_buffer.raw(),
1011 usage: hal::StateTransition {
1012 from: wgt::BufferUses::MAP_WRITE,
1013 to: wgt::BufferUses::COPY_SRC,
1014 },
1015 };
1016 let transition_dst = hal::BufferBarrier::<dyn hal::DynBuffer> {
1017 buffer: raw_buf,
1018 usage: hal::StateTransition {
1019 from: wgt::BufferUses::empty(),
1020 to: wgt::BufferUses::COPY_DST,
1021 },
1022 };
1023 let mut pending_writes = queue.pending_writes.lock();
1024 let encoder = pending_writes.activate();
1025 unsafe {
1026 encoder.transition_buffers(&[transition_src, transition_dst]);
1027 if self.size > 0 {
1028 encoder.copy_buffer_to_buffer(
1029 staging_buffer.raw(),
1030 raw_buf,
1031 region.as_slice(),
1032 );
1033 }
1034 }
1035 pending_writes.consume(staging_buffer);
1036 pending_writes.insert_buffer(self);
1037 }
1038 }
1039 BufferMapState::Idle => {
1040 return Err(BufferAccessError::NotMapped);
1041 }
1042 BufferMapState::Waiting(pending) => {
1043 return Ok(Some((pending.op, Err(BufferAccessError::MapAborted))));
1044 }
1045 BufferMapState::Active {
1046 mapping,
1047 range,
1048 host,
1049 } => {
1050 if host == HostMap::Write {
1051 #[cfg(feature = "trace")]
1052 if let Some(ref mut trace) = *device.trace.lock() {
1053 use crate::device::trace::{DataKind, IntoTrace};
1054
1055 let size = range.end - range.start;
1056 let data = trace.make_binary(DataKind::Bin, unsafe {
1057 core::slice::from_raw_parts(mapping.ptr.as_ptr(), size as usize)
1058 });
1059 trace.add(trace::Action::WriteBuffer {
1060 id: self.to_trace(),
1061 data,
1062 offset: range.start,
1063 size,
1064 queued: false,
1065 });
1066 }
1067 if !mapping.is_coherent {
1068 unsafe { device.raw().flush_mapped_ranges(raw_buf, &[range]) };
1069 }
1070 }
1071 unsafe { device.raw().unmap_buffer(raw_buf) };
1072 }
1073 }
1074 Ok(None)
1075 }
1076
1077 pub fn destroy(self: &Arc<Self>) {
1078 let device = &self.device;
1079
1080 let destroyed = {
1081 let mut snatch_guard = device.snatchable_lock.write();
1082
1083 let raw = match self.raw.snatch(&mut snatch_guard) {
1084 Some(raw) => raw,
1085 None => {
1086 return;
1088 }
1089 };
1090
1091 let timestamp_normalization_bind_group = self
1092 .timestamp_normalization_bind_group
1093 .snatch(&mut snatch_guard);
1094
1095 let indirect_validation_bind_groups = self
1096 .indirect_validation_bind_groups
1097 .snatch(&mut snatch_guard);
1098
1099 drop(snatch_guard);
1100
1101 let bind_groups = {
1102 let mut guard = self.bind_groups.lock();
1103 mem::take(&mut *guard)
1104 };
1105
1106 DestroyedBuffer {
1107 raw: ManuallyDrop::new(raw),
1108 device: Arc::clone(&self.device),
1109 label: self.label().to_owned(),
1110 bind_groups,
1111 timestamp_normalization_bind_group,
1112 indirect_validation_bind_groups,
1113 }
1114 };
1115
1116 if matches!(*self.map_state.lock(), BufferMapState::Active { .. }) {
1117 let _ = self.destroyed_while_mapped.set(destroyed);
1118 return;
1119 }
1120 let temp = queue::TempResource::DestroyedBuffer(destroyed);
1121
1122 let Some(queue) = device.get_queue() else {
1123 return;
1124 };
1125
1126 {
1127 let mut pending_writes = queue.pending_writes.lock();
1128 if pending_writes.contains_buffer(self) {
1129 pending_writes.consume_temp(temp);
1130 return;
1131 }
1132 }
1133
1134 let mut life_lock = queue.lock_life();
1135 let last_submit_index = life_lock.get_buffer_latest_submission_index(self);
1136 if let Some(last_submit_index) = last_submit_index {
1137 life_lock.schedule_resource_destruction(temp, last_submit_index);
1138 }
1139 }
1140}
1141
1142#[derive(Clone, Debug, Error)]
1143#[non_exhaustive]
1144pub enum CreateBufferError {
1145 #[error(transparent)]
1146 Device(#[from] DeviceError),
1147 #[error("Failed to map buffer while creating: {0}")]
1148 AccessError(#[from] BufferAccessError),
1149 #[error("Buffers that are mapped at creation have to be aligned to `COPY_BUFFER_ALIGNMENT`")]
1150 UnalignedSize,
1151 #[error("Invalid usage flags {0:?}")]
1152 InvalidUsage(wgt::BufferUsages),
1153 #[error("`MAP` usage can only be combined with the opposite `COPY`, requested {0:?}")]
1154 UsageMismatch(wgt::BufferUsages),
1155 #[error("Buffer size {requested} is greater than the maximum buffer size ({maximum})")]
1156 MaxBufferSize { requested: u64, maximum: u64 },
1157 #[error(transparent)]
1158 MissingDownlevelFlags(#[from] MissingDownlevelFlags),
1159 #[error(transparent)]
1160 MissingFeatures(#[from] MissingFeatures),
1161 #[error("Failed to create bind group for indirect buffer validation: {0}")]
1162 IndirectValidationBindGroup(DeviceError),
1163}
1164
1165crate::impl_resource_type!(Buffer);
1166crate::impl_labeled!(Buffer);
1167crate::impl_parent_device!(Buffer);
1168crate::impl_storage_item!(Buffer);
1169crate::impl_trackable!(Buffer);
1170
1171impl WebGpuError for CreateBufferError {
1172 fn webgpu_error_type(&self) -> ErrorType {
1173 match self {
1174 Self::Device(e) => e.webgpu_error_type(),
1175 Self::AccessError(e) => e.webgpu_error_type(),
1176 Self::MissingDownlevelFlags(e) => e.webgpu_error_type(),
1177 Self::IndirectValidationBindGroup(e) => e.webgpu_error_type(),
1178 Self::MissingFeatures(e) => e.webgpu_error_type(),
1179
1180 Self::UnalignedSize
1181 | Self::InvalidUsage(_)
1182 | Self::UsageMismatch(_)
1183 | Self::MaxBufferSize { .. } => ErrorType::Validation,
1184 }
1185 }
1186}
1187
1188#[derive(Debug)]
1190pub struct DestroyedBuffer {
1191 raw: ManuallyDrop<Box<dyn hal::DynBuffer>>,
1192 device: Arc<Device>,
1193 label: String,
1194 bind_groups: WeakVec<BindGroup>,
1195 timestamp_normalization_bind_group: Option<TimestampNormalizationBindGroup>,
1196 indirect_validation_bind_groups: Option<crate::indirect_validation::BindGroups>,
1197}
1198
1199impl DestroyedBuffer {
1200 pub fn label(&self) -> &dyn fmt::Debug {
1201 &self.label
1202 }
1203}
1204
1205impl Drop for DestroyedBuffer {
1206 fn drop(&mut self) {
1207 let mut deferred = self.device.deferred_destroy.lock();
1208 deferred.push(DeferredDestroy::BindGroups(mem::take(
1209 &mut self.bind_groups,
1210 )));
1211 drop(deferred);
1212
1213 if let Some(raw) = self.timestamp_normalization_bind_group.take() {
1214 raw.dispose(self.device.raw());
1215 }
1216
1217 if let Some(raw) = self.indirect_validation_bind_groups.take() {
1218 raw.dispose(self.device.raw());
1219 }
1220
1221 resource_log!("Destroy raw Buffer (destroyed) {:?}", self.label());
1222 let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1224 unsafe {
1225 hal::DynDevice::destroy_buffer(self.device.raw(), raw);
1226 }
1227 }
1228}
1229
1230#[cfg(send_sync)]
1231unsafe impl Send for StagingBuffer {}
1232#[cfg(send_sync)]
1233unsafe impl Sync for StagingBuffer {}
1234
1235#[derive(Debug)]
1255pub struct StagingBuffer {
1256 raw: Box<dyn hal::DynBuffer>,
1257 device: Arc<Device>,
1258 pub(crate) size: wgt::BufferSize,
1259 is_coherent: bool,
1260 ptr: NonNull<u8>,
1261}
1262
1263impl StagingBuffer {
1264 pub(crate) fn new(device: &Arc<Device>, size: wgt::BufferSize) -> Result<Self, DeviceError> {
1265 profiling::scope!("StagingBuffer::new");
1266 let stage_desc = hal::BufferDescriptor {
1267 label: hal_label(Some("(wgpu internal) Staging"), device.instance_flags),
1268 size: size.get(),
1269 usage: wgt::BufferUses::MAP_WRITE | wgt::BufferUses::COPY_SRC,
1270 memory_flags: hal::MemoryFlags::TRANSIENT,
1271 };
1272
1273 let raw = unsafe { device.raw().create_buffer(&stage_desc) }
1274 .map_err(|e| device.handle_hal_error(e))?;
1275 let mapping = unsafe { device.raw().map_buffer(raw.as_ref(), 0..size.get()) }
1276 .map_err(|e| device.handle_hal_error(e))?;
1277
1278 let staging_buffer = StagingBuffer {
1279 raw,
1280 device: device.clone(),
1281 size,
1282 is_coherent: mapping.is_coherent,
1283 ptr: mapping.ptr,
1284 };
1285
1286 Ok(staging_buffer)
1287 }
1288
1289 pub(crate) unsafe fn ptr(&self) -> NonNull<u8> {
1292 self.ptr
1293 }
1294
1295 #[cfg(feature = "trace")]
1296 pub(crate) fn get_data(&self) -> &[u8] {
1297 unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.size.get() as usize) }
1298 }
1299
1300 pub(crate) fn write_zeros(&mut self) {
1301 unsafe { core::ptr::write_bytes(self.ptr.as_ptr(), 0, self.size.get() as usize) };
1302 }
1303
1304 pub(crate) fn write(&mut self, data: &[u8]) {
1305 assert!(data.len() >= self.size.get() as usize);
1306 unsafe {
1309 core::ptr::copy_nonoverlapping(
1310 data.as_ptr(),
1311 self.ptr.as_ptr(),
1312 self.size.get() as usize,
1313 );
1314 }
1315 }
1316
1317 pub(crate) unsafe fn write_with_offset(
1319 &mut self,
1320 data: &[u8],
1321 src_offset: isize,
1322 dst_offset: isize,
1323 size: usize,
1324 ) {
1325 unsafe {
1326 debug_assert!(
1327 (src_offset + size as isize) as usize <= data.len(),
1328 "src_offset + size must be in-bounds: src_offset = {}, size = {}, data.len() = {}",
1329 src_offset,
1330 size,
1331 data.len()
1332 );
1333 core::ptr::copy_nonoverlapping(
1334 data.as_ptr().offset(src_offset),
1335 self.ptr.as_ptr().offset(dst_offset),
1336 size,
1337 );
1338 }
1339 }
1340
1341 pub(crate) fn flush(self) -> FlushedStagingBuffer {
1342 let device = self.device.raw();
1343 if !self.is_coherent {
1344 #[allow(clippy::single_range_in_vec_init)]
1345 unsafe {
1346 device.flush_mapped_ranges(self.raw.as_ref(), &[0..self.size.get()])
1347 };
1348 }
1349 unsafe { device.unmap_buffer(self.raw.as_ref()) };
1350
1351 let StagingBuffer {
1352 raw, device, size, ..
1353 } = self;
1354
1355 FlushedStagingBuffer {
1356 raw: ManuallyDrop::new(raw),
1357 device,
1358 size,
1359 }
1360 }
1361}
1362
1363crate::impl_resource_type!(StagingBuffer);
1364crate::impl_storage_item!(StagingBuffer);
1365
1366#[derive(Debug)]
1367pub struct FlushedStagingBuffer {
1368 raw: ManuallyDrop<Box<dyn hal::DynBuffer>>,
1369 device: Arc<Device>,
1370 pub(crate) size: wgt::BufferSize,
1371}
1372
1373impl FlushedStagingBuffer {
1374 pub(crate) fn raw(&self) -> &dyn hal::DynBuffer {
1375 self.raw.as_ref()
1376 }
1377}
1378
1379impl Drop for FlushedStagingBuffer {
1380 fn drop(&mut self) {
1381 resource_log!("Destroy raw StagingBuffer");
1382 let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1384 unsafe { self.device.raw().destroy_buffer(raw) };
1385 }
1386}
1387
1388pub type TextureDescriptor<'a> = wgt::TextureDescriptor<Label<'a>, Vec<wgt::TextureFormat>>;
1389
1390#[derive(Debug)]
1391pub(crate) enum TextureInner {
1392 Native {
1393 raw: Box<dyn hal::DynTexture>,
1394 },
1395 Surface {
1396 raw: Box<dyn hal::DynSurfaceTexture>,
1397 },
1398}
1399
1400impl TextureInner {
1401 pub(crate) fn raw(&self) -> &dyn hal::DynTexture {
1402 match self {
1403 Self::Native { raw } => raw.as_ref(),
1404 Self::Surface { raw, .. } => raw.as_ref().borrow(),
1405 }
1406 }
1407}
1408
1409#[derive(Debug)]
1410pub enum TextureClearMode {
1411 BufferCopy,
1412 RenderPass {
1414 clear_views: SmallVec<[ManuallyDrop<Box<dyn hal::DynTextureView>>; 1]>,
1415 is_color: bool,
1416 },
1417 Surface {
1418 clear_view: ManuallyDrop<Box<dyn hal::DynTextureView>>,
1419 },
1420 None,
1423}
1424
1425#[derive(Debug)]
1426pub struct Texture {
1427 pub(crate) inner: Snatchable2<TextureInner>,
1428 pub(crate) device: Arc<Device>,
1429 pub(crate) desc: wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
1430 pub(crate) _hal_usage: wgt::TextureUses,
1431 pub(crate) format_features: wgt::TextureFormatFeatures,
1432 pub(crate) initialization_status: RwLock<TextureInitTracker>,
1433 pub(crate) full_range: TextureSelector,
1434 pub(crate) tracking_data: TrackingData,
1435 pub(crate) clear_mode: RwLock<TextureClearMode>,
1436 pub(crate) views: Mutex<WeakVec<TextureView>>,
1437 pub(crate) bind_groups: Mutex<WeakVec<BindGroup>>,
1439}
1440
1441impl Texture {
1442 pub(crate) fn new(
1443 device: &Arc<Device>,
1444 inner: TextureInner,
1445 hal_usage: wgt::TextureUses,
1446 desc: &TextureDescriptor,
1447 format_features: wgt::TextureFormatFeatures,
1448 clear_mode: TextureClearMode,
1449 init: bool,
1450 ) -> Self {
1451 Texture {
1452 inner: Snatchable2::new(inner),
1453 device: device.clone(),
1454 desc: desc.map_label(|label| label.to_string()),
1455 _hal_usage: hal_usage,
1456 format_features,
1457 initialization_status: RwLock::new(
1458 rank::TEXTURE_INITIALIZATION_STATUS,
1459 if init {
1460 TextureInitTracker::new(desc.mip_level_count, desc.array_layer_count())
1461 } else {
1462 TextureInitTracker::new(desc.mip_level_count, 0)
1463 },
1464 ),
1465 full_range: TextureSelector {
1466 mips: 0..desc.mip_level_count,
1467 layers: 0..desc.array_layer_count(),
1468 },
1469 tracking_data: TrackingData::new(device.tracker_indices.textures.clone()),
1470 clear_mode: RwLock::new(rank::TEXTURE_CLEAR_MODE, clear_mode),
1471 views: Mutex::new(rank::TEXTURE_VIEWS, WeakVec::new()),
1472 bind_groups: Mutex::new(rank::TEXTURE_BIND_GROUPS, WeakVec::new()),
1473 }
1474 }
1475
1476 pub(crate) fn invalid(device: &Arc<Device>, desc: &TextureDescriptor) -> Self {
1477 Texture {
1478 inner: Snatchable2::invalid(),
1479 device: device.clone(),
1480 desc: desc.map_label(|label| label.to_string()),
1481 _hal_usage: wgt::TextureUses::empty(),
1482 format_features: wgt::TextureFormatFeatures {
1483 allowed_usages: wgt::TextureUsages::empty(),
1484 flags: wgt::TextureFormatFeatureFlags::empty(),
1485 },
1486 initialization_status: RwLock::new(
1487 rank::TEXTURE_INITIALIZATION_STATUS,
1488 TextureInitTracker::new(0, 0),
1489 ),
1490 full_range: TextureSelector {
1491 mips: 0..desc.mip_level_count,
1492 layers: 0..desc.array_layer_count(),
1493 },
1494 tracking_data: TrackingData::new(device.tracker_indices.textures.clone()),
1495 clear_mode: RwLock::new(rank::TEXTURE_CLEAR_MODE, TextureClearMode::None),
1496 views: Mutex::new(rank::TEXTURE_VIEWS, WeakVec::new()),
1497 bind_groups: Mutex::new(rank::TEXTURE_BIND_GROUPS, WeakVec::new()),
1498 }
1499 }
1500
1501 pub(crate) fn check_usage(
1504 &self,
1505 expected: wgt::TextureUsages,
1506 ) -> Result<(), MissingTextureUsageError> {
1507 if self.desc.usage.contains(expected) {
1508 Ok(())
1509 } else {
1510 Err(MissingTextureUsageError {
1511 res: self.error_ident(),
1512 actual: self.desc.usage,
1513 expected,
1514 })
1515 }
1516 }
1517}
1518
1519impl Drop for Texture {
1520 fn drop(&mut self) {
1521 #[cfg(feature = "trace")]
1522 {
1523 let mut t = self.device.trace.lock();
1524 if let Some(t) = t.as_mut() {
1525 use crate::device::trace::to_trace;
1526
1527 t.add(trace::Action::DropTexture(unsafe { to_trace(self) }));
1529 }
1530 }
1531 match *self.clear_mode.write() {
1532 TextureClearMode::Surface {
1533 ref mut clear_view, ..
1534 } => {
1535 let raw = unsafe { ManuallyDrop::take(clear_view) };
1537 unsafe {
1538 self.device.raw().destroy_texture_view(raw);
1539 }
1540 }
1541 TextureClearMode::RenderPass {
1542 ref mut clear_views,
1543 ..
1544 } => {
1545 clear_views.iter_mut().for_each(|clear_view| {
1546 let raw = unsafe { ManuallyDrop::take(clear_view) };
1548 unsafe {
1549 self.device.raw().destroy_texture_view(raw);
1550 }
1551 });
1552 }
1553 _ => {}
1554 };
1555
1556 if let Some(TextureInner::Native { raw }) = self.inner.take().maybe_valid() {
1557 resource_log!("Destroy raw {}", self.error_ident());
1558 unsafe {
1559 self.device.raw().destroy_texture(raw);
1560 }
1561 }
1562 }
1563}
1564
1565impl RawResourceAccess for Texture {
1566 type DynResource = dyn hal::DynTexture;
1567
1568 fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
1569 self.inner.get(guard).maybe_valid().map(|t| t.raw())
1570 }
1571}
1572
1573impl Texture {
1574 pub(crate) fn try_inner<'a>(
1575 &'a self,
1576 guard: &'a SnatchGuard,
1577 ) -> Result<&'a TextureInner, InvalidOrDestroyedResourceError> {
1578 match self.inner.get(guard) {
1579 DestructibleResourceState::Valid(t) => Ok(t),
1580 DestructibleResourceState::Invalid => {
1581 Err(InvalidOrDestroyedResourceError::InvalidResource(
1582 InvalidResourceError(self.error_ident()),
1583 ))
1584 }
1585 DestructibleResourceState::Destroyed => {
1586 Err(InvalidOrDestroyedResourceError::DestroyedResource(
1587 DestroyedResourceError(self.error_ident()),
1588 ))
1589 }
1590 }
1591 }
1592
1593 pub(crate) fn check_destroyed(
1594 &self,
1595 guard: &SnatchGuard,
1596 ) -> Result<(), DestroyedResourceError> {
1597 match self.inner.get(guard) {
1598 DestructibleResourceState::Valid(_) => Ok(()),
1599 DestructibleResourceState::Invalid => Ok(()),
1600 DestructibleResourceState::Destroyed => Err(DestroyedResourceError(self.error_ident())),
1601 }
1602 }
1603
1604 pub(crate) fn check_valid(&self, guard: &SnatchGuard) -> Result<(), InvalidResourceError> {
1605 match self.inner.get(guard) {
1606 DestructibleResourceState::Valid(_) => Ok(()),
1607 DestructibleResourceState::Invalid => Err(InvalidResourceError(self.error_ident())),
1608 DestructibleResourceState::Destroyed => Ok(()),
1609 }
1610 }
1611
1612 pub(crate) fn get_clear_view<'a>(
1613 clear_mode: &'a TextureClearMode,
1614 desc: &'a wgt::TextureDescriptor<String, Vec<wgt::TextureFormat>>,
1615 mip_level: u32,
1616 depth_or_layer: u32,
1617 ) -> &'a dyn hal::DynTextureView {
1618 match *clear_mode {
1619 TextureClearMode::BufferCopy => {
1620 panic!("Given texture is cleared with buffer copies, not render passes")
1621 }
1622 TextureClearMode::None => {
1623 panic!("Given texture can't be cleared")
1624 }
1625 TextureClearMode::Surface { ref clear_view, .. } => clear_view.as_ref(),
1626 TextureClearMode::RenderPass {
1627 ref clear_views, ..
1628 } => {
1629 let index = if desc.dimension == wgt::TextureDimension::D3 {
1630 (0..mip_level).fold(0, |acc, mip| {
1631 acc + (desc.size.depth_or_array_layers >> mip).max(1)
1632 })
1633 } else {
1634 mip_level * desc.size.depth_or_array_layers
1635 } + depth_or_layer;
1636 clear_views[index as usize].as_ref()
1637 }
1638 }
1639 }
1640
1641 pub fn destroy(self: &Arc<Self>) {
1642 let device = &self.device;
1643
1644 let temp = {
1645 let raw = match self
1646 .inner
1647 .snatch(&mut device.snatchable_lock.write())
1648 .maybe_valid()
1649 {
1650 Some(TextureInner::Native { raw }) => raw,
1651 Some(TextureInner::Surface { .. }) => {
1652 return;
1653 }
1654 None => {
1655 return;
1657 }
1658 };
1659
1660 let views = {
1661 let mut guard = self.views.lock();
1662 mem::take(&mut *guard)
1663 };
1664
1665 let bind_groups = {
1666 let mut guard = self.bind_groups.lock();
1667 mem::take(&mut *guard)
1668 };
1669
1670 queue::TempResource::DestroyedTexture(DestroyedTexture {
1671 raw: ManuallyDrop::new(raw),
1672 views,
1673 clear_mode: mem::replace(&mut *self.clear_mode.write(), TextureClearMode::None),
1674 bind_groups,
1675 device: Arc::clone(&self.device),
1676 label: self.label().to_owned(),
1677 })
1678 };
1679
1680 let Some(queue) = device.get_queue() else {
1681 return;
1682 };
1683
1684 {
1685 let mut pending_writes = queue.pending_writes.lock();
1686 if pending_writes.contains_texture(self) {
1687 pending_writes.consume_temp(temp);
1688 return;
1689 }
1690 }
1691
1692 let mut life_lock = queue.lock_life();
1693 let last_submit_index = life_lock.get_texture_latest_submission_index(self);
1694 if let Some(last_submit_index) = last_submit_index {
1695 life_lock.schedule_resource_destruction(temp, last_submit_index);
1696 }
1697 }
1698}
1699
1700#[derive(Debug)]
1702pub struct DestroyedTexture {
1703 raw: ManuallyDrop<Box<dyn hal::DynTexture>>,
1704 views: WeakVec<TextureView>,
1705 clear_mode: TextureClearMode,
1706 bind_groups: WeakVec<BindGroup>,
1707 device: Arc<Device>,
1708 label: String,
1709}
1710
1711impl DestroyedTexture {
1712 pub fn label(&self) -> &dyn fmt::Debug {
1713 &self.label
1714 }
1715}
1716
1717impl Drop for DestroyedTexture {
1718 fn drop(&mut self) {
1719 let device = &self.device;
1720
1721 let mut deferred = device.deferred_destroy.lock();
1722 deferred.push(DeferredDestroy::TextureViews(mem::take(&mut self.views)));
1723 deferred.push(DeferredDestroy::BindGroups(mem::take(
1724 &mut self.bind_groups,
1725 )));
1726 drop(deferred);
1727
1728 match mem::replace(&mut self.clear_mode, TextureClearMode::None) {
1729 TextureClearMode::RenderPass { clear_views, .. } => {
1730 for clear_view in clear_views {
1731 let raw = ManuallyDrop::into_inner(clear_view);
1732 unsafe { self.device.raw().destroy_texture_view(raw) };
1733 }
1734 }
1735 TextureClearMode::Surface { clear_view } => {
1736 let raw = ManuallyDrop::into_inner(clear_view);
1737 unsafe { self.device.raw().destroy_texture_view(raw) };
1738 }
1739 _ => (),
1740 }
1741
1742 resource_log!("Destroy raw Texture (destroyed) {:?}", self.label());
1743 let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
1745 unsafe {
1746 self.device.raw().destroy_texture(raw);
1747 }
1748 }
1749}
1750
1751#[derive(Clone, Copy, Debug)]
1752pub enum TextureErrorDimension {
1753 X,
1754 Y,
1755 Z,
1756}
1757
1758#[derive(Clone, Debug, Error)]
1759#[non_exhaustive]
1760pub enum TextureDimensionError {
1761 #[error("Dimension {0:?} is zero")]
1762 Zero(TextureErrorDimension),
1763 #[error("Dimension {dim:?} value {given} exceeds the limit of {limit}")]
1764 LimitExceeded {
1765 dim: TextureErrorDimension,
1766 given: u32,
1767 limit: u32,
1768 },
1769 #[error("Sample count {0} is invalid")]
1770 InvalidSampleCount(u32),
1771 #[error("Width {width} is not a multiple of {format:?}'s block width ({block_width})")]
1772 NotMultipleOfBlockWidth {
1773 width: u32,
1774 block_width: u32,
1775 format: wgt::TextureFormat,
1776 },
1777 #[error("Height {height} is not a multiple of {format:?}'s block height ({block_height})")]
1778 NotMultipleOfBlockHeight {
1779 height: u32,
1780 block_height: u32,
1781 format: wgt::TextureFormat,
1782 },
1783 #[error(
1784 "Width {width} is not a multiple of {format:?}'s width multiple requirement ({multiple})"
1785 )]
1786 WidthNotMultipleOf {
1787 width: u32,
1788 multiple: u32,
1789 format: wgt::TextureFormat,
1790 },
1791 #[error("Height {height} is not a multiple of {format:?}'s height multiple requirement ({multiple})")]
1792 HeightNotMultipleOf {
1793 height: u32,
1794 multiple: u32,
1795 format: wgt::TextureFormat,
1796 },
1797 #[error("Multisampled texture depth or array layers must be 1, got {0}")]
1798 MultisampledDepthOrArrayLayer(u32),
1799}
1800
1801impl WebGpuError for TextureDimensionError {
1802 fn webgpu_error_type(&self) -> ErrorType {
1803 ErrorType::Validation
1804 }
1805}
1806
1807#[derive(Clone, Debug, Error)]
1808#[non_exhaustive]
1809pub enum CreateTextureError {
1810 #[error(transparent)]
1811 Device(#[from] DeviceError),
1812 #[error(transparent)]
1813 CreateTextureView(#[from] CreateTextureViewError),
1814 #[error("Invalid usage flags {0:?}")]
1815 InvalidUsage(wgt::TextureUsages),
1816 #[error(transparent)]
1817 InvalidDimension(#[from] TextureDimensionError),
1818 #[error("Depth texture ({1:?}) can't be created as {0:?}")]
1819 InvalidDepthDimension(wgt::TextureDimension, wgt::TextureFormat),
1820 #[error("Compressed texture ({1:?}) can't be created as {0:?}")]
1821 InvalidCompressedDimension(wgt::TextureDimension, wgt::TextureFormat),
1822 #[error(
1823 "Texture descriptor mip level count {requested} is invalid, maximum allowed is {maximum}"
1824 )]
1825 InvalidMipLevelCount { requested: u32, maximum: u32 },
1826 #[error(
1827 "Texture usages {0:?} are not allowed on a texture of type {1:?}{downlevel_suffix}",
1828 downlevel_suffix = if *.2 { " due to downlevel restrictions" } else { "" }
1829 )]
1830 InvalidFormatUsages(wgt::TextureUsages, wgt::TextureFormat, bool),
1831 #[error("The view format {0:?} is not compatible with texture format {1:?}, only changing srgb-ness is allowed.")]
1832 InvalidViewFormat(wgt::TextureFormat, wgt::TextureFormat),
1833 #[error("Transient texture usage must be equal to `TRANSIENT_ATTACHMENT | RENDER_ATTACHMENT`, but got `{0:?}`")]
1834 InvalidTransientTextureUsage(wgt::TextureUsages),
1835 #[error("Transient texture view formats must be empty")]
1836 InvalidTransientTextureViewFormats,
1837 #[error("Texture usages {0:?} are not allowed on a texture of dimensions {1:?}")]
1838 InvalidDimensionUsages(wgt::TextureUsages, wgt::TextureDimension),
1839 #[error("Texture usage STORAGE_BINDING is not allowed for multisampled textures")]
1840 InvalidMultisampledStorageBinding,
1841 #[error("Format {0:?} does not support multisampling")]
1842 InvalidMultisampledFormat(wgt::TextureFormat),
1843 #[error("Sample count {0} is not supported by format {1:?} on this device. The WebGPU spec guarantees {2:?} samples are supported by this format. With the TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES feature your device supports {3:?}.")]
1844 InvalidSampleCount(u32, wgt::TextureFormat, Vec<u32>, Vec<u32>),
1845 #[error("Multisampled textures must have RENDER_ATTACHMENT usage")]
1846 MultisampledNotRenderAttachment,
1847 #[error("Transient texture mip level count ({0}) must be 1")]
1848 InvalidTransientTextureMipLevelCount(u32),
1849 #[error("Transient texture layer count ({0}) must be 1")]
1850 InvalidTransientTextureLayerCount(u32),
1851 #[error("Texture format {0:?} can't be used due to missing features")]
1852 MissingFeatures(wgt::TextureFormat, #[source] MissingFeatures),
1853 #[error(transparent)]
1854 MissingDownlevelFlags(#[from] MissingDownlevelFlags),
1855}
1856
1857crate::impl_resource_type!(Texture);
1858impl Labeled for Texture {
1859 fn label(&self) -> &str {
1860 &self.desc.label
1861 }
1862}
1863crate::impl_parent_device!(Texture);
1864crate::impl_storage_item!(Texture);
1865crate::impl_trackable!(Texture);
1866
1867impl Borrow<TextureSelector> for Texture {
1868 fn borrow(&self) -> &TextureSelector {
1869 &self.full_range
1870 }
1871}
1872
1873impl WebGpuError for CreateTextureError {
1874 fn webgpu_error_type(&self) -> ErrorType {
1875 match self {
1876 Self::Device(e) => e.webgpu_error_type(),
1877 Self::CreateTextureView(e) => e.webgpu_error_type(),
1878 Self::InvalidDimension(e) => e.webgpu_error_type(),
1879 Self::MissingFeatures(_, e) => e.webgpu_error_type(),
1880 Self::MissingDownlevelFlags(e) => e.webgpu_error_type(),
1881
1882 Self::InvalidUsage(_)
1883 | Self::InvalidDepthDimension(_, _)
1884 | Self::InvalidCompressedDimension(_, _)
1885 | Self::InvalidMipLevelCount { .. }
1886 | Self::InvalidFormatUsages(_, _, _)
1887 | Self::InvalidViewFormat(_, _)
1888 | Self::InvalidDimensionUsages(_, _)
1889 | Self::InvalidMultisampledStorageBinding
1890 | Self::InvalidMultisampledFormat(_)
1891 | Self::InvalidSampleCount(..)
1892 | Self::InvalidTransientTextureUsage(_)
1893 | Self::InvalidTransientTextureMipLevelCount(_)
1894 | Self::InvalidTransientTextureLayerCount(_)
1895 | Self::InvalidTransientTextureViewFormats
1896 | Self::MultisampledNotRenderAttachment => ErrorType::Validation,
1897 }
1898 }
1899}
1900
1901#[derive(Clone, Debug, Default, Eq, PartialEq)]
1903#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
1904#[cfg_attr(feature = "serde", serde(default))]
1905pub struct TextureViewDescriptor<'a> {
1906 pub label: Label<'a>,
1910 pub format: Option<wgt::TextureFormat>,
1915 pub dimension: Option<wgt::TextureViewDimension>,
1921 pub usage: Option<wgt::TextureUsages>,
1924 pub range: wgt::ImageSubresourceRange,
1926}
1927
1928#[derive(Debug)]
1929pub(crate) struct HalTextureViewDescriptor {
1930 pub texture_format: wgt::TextureFormat,
1931 pub format: wgt::TextureFormat,
1932 pub usage: wgt::TextureUsages,
1933 pub dimension: wgt::TextureViewDimension,
1934 pub range: wgt::ImageSubresourceRange,
1935}
1936
1937impl HalTextureViewDescriptor {
1938 pub fn aspects(&self) -> hal::FormatAspects {
1939 hal::FormatAspects::new(self.texture_format, self.range.aspect)
1940 }
1941}
1942
1943#[derive(Debug, Copy, Clone, Error)]
1944pub enum TextureViewNotRenderableReason {
1945 #[error("The texture this view references doesn't include the RENDER_ATTACHMENT usage. Provided usages: {0:?}")]
1946 Usage(wgt::TextureUsages),
1947 #[error("The dimension of this texture view is not 2D. View dimension: {0:?}")]
1948 Dimension(wgt::TextureViewDimension),
1949 #[error("This texture view has more than one mipmap level. View mipmap levels: {0:?}")]
1950 MipLevelCount(u32),
1951 #[error("This texture view has more than one array layer. View array layers: {0:?}")]
1952 ArrayLayerCount(u32),
1953 #[error(
1954 "The aspects of this texture view are a subset of the aspects in the original texture. Aspects: {0:?}"
1955 )]
1956 Aspects(hal::FormatAspects),
1957}
1958
1959#[derive(Debug)]
1960pub struct TextureView {
1961 pub(crate) raw: Snatchable<Box<dyn hal::DynTextureView>>,
1962 pub(crate) parent: Arc<Texture>,
1964 pub(crate) device: Arc<Device>,
1965 pub(crate) desc: HalTextureViewDescriptor,
1966 pub(crate) format_features: wgt::TextureFormatFeatures,
1967 pub(crate) render_extent: Result<wgt::Extent3d, TextureViewNotRenderableReason>,
1969 pub(crate) samples: u32,
1970 pub(crate) selector: TextureSelector,
1971 pub(crate) label: String,
1973}
1974
1975impl Drop for TextureView {
1976 fn drop(&mut self) {
1977 if let Some(raw) = self.raw.take() {
1978 resource_log!("Destroy raw {}", self.error_ident());
1979 unsafe {
1980 self.device.raw().destroy_texture_view(raw);
1981 }
1982 }
1983 }
1984}
1985
1986impl RawResourceAccess for TextureView {
1987 type DynResource = dyn hal::DynTextureView;
1988
1989 fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
1990 self.raw.get(guard).map(|it| it.as_ref())
1991 }
1992
1993 fn try_raw<'a>(
1994 &'a self,
1995 guard: &'a SnatchGuard,
1996 ) -> Result<&'a Self::DynResource, DestroyedResourceError> {
1997 self.parent.check_destroyed(guard)?;
1998
1999 self.raw(guard)
2000 .ok_or_else(|| DestroyedResourceError(self.error_ident()))
2001 }
2002}
2003
2004impl TextureView {
2005 pub(crate) fn check_usage(
2008 &self,
2009 expected: wgt::TextureUsages,
2010 ) -> Result<(), MissingTextureUsageError> {
2011 if self.desc.usage.contains(expected) {
2012 Ok(())
2013 } else {
2014 Err(MissingTextureUsageError {
2015 res: self.error_ident(),
2016 actual: self.desc.usage,
2017 expected,
2018 })
2019 }
2020 }
2021}
2022
2023#[derive(Clone, Debug, Error)]
2024#[non_exhaustive]
2025pub enum CreateTextureViewError {
2026 #[error(transparent)]
2027 Device(#[from] DeviceError),
2028 #[error(transparent)]
2029 DestroyedResource(#[from] DestroyedResourceError),
2030 #[error("Invalid texture view dimension `{view:?}` with texture of dimension `{texture:?}`")]
2031 InvalidTextureViewDimension {
2032 view: wgt::TextureViewDimension,
2033 texture: wgt::TextureDimension,
2034 },
2035 #[error("Texture view format `{0:?}` cannot be used as a render attachment. Make sure the format supports RENDER_ATTACHMENT usage and required device features are enabled.")]
2036 TextureViewFormatNotRenderable(wgt::TextureFormat),
2037 #[error("Texture view format `{0:?}` cannot be used as a storage binding. Make sure the format supports STORAGE usage and required device features are enabled.")]
2038 TextureViewFormatNotStorage(wgt::TextureFormat),
2039 #[error("Texture view usages (`{view:?}`) must be a subset of the texture's original usages (`{texture:?}`)")]
2040 InvalidTextureViewUsage {
2041 view: wgt::TextureUsages,
2042 texture: wgt::TextureUsages,
2043 },
2044 #[error("Texture view dimension `{0:?}` cannot be used with a multisampled texture")]
2045 InvalidMultisampledTextureViewDimension(wgt::TextureViewDimension),
2046 #[error(
2047 "TextureView has an arrayLayerCount of {depth}. Views of type `Cube` must have arrayLayerCount of 6."
2048 )]
2049 InvalidCubemapTextureDepth { depth: u32 },
2050 #[error("TextureView has an arrayLayerCount of {depth}. Views of type `CubeArray` must have an arrayLayerCount that is a multiple of 6.")]
2051 InvalidCubemapArrayTextureDepth { depth: u32 },
2052 #[error("Source texture width and height must be equal for a texture view of dimension `Cube`/`CubeArray`")]
2053 InvalidCubeTextureViewSize,
2054 #[error("Mip level count is 0")]
2055 ZeroMipLevelCount,
2056 #[error("Array layer count is 0")]
2057 ZeroArrayLayerCount,
2058 #[error(
2059 "`TextureView` starts at mip level {base_mip_level} and spans {mip_level_count} mip \
2060 levels, but the texture view only has {total} total mip level(s)"
2061 )]
2062 TooManyMipLevels {
2063 base_mip_level: u32,
2064 mip_level_count: u32,
2065 total: u32,
2066 },
2067 #[error(
2068 "`TextureView` starts at array layer {base_array_layer} and spans {array_layer_count}) \
2069 array layers, but the texture view only has {total} total layer(s)"
2070 )]
2071 TooManyArrayLayers {
2072 base_array_layer: u32,
2073 array_layer_count: u32,
2074 total: u32,
2075 },
2076 #[error("Requested array layer count {requested} is not valid for the target view dimension {dim:?}")]
2077 InvalidArrayLayerCount {
2078 requested: u32,
2079 dim: wgt::TextureViewDimension,
2080 },
2081 #[error(
2082 "Aspect {requested_aspect:?} is not a valid aspect of the source texture format {texture_format:?}"
2083 )]
2084 InvalidAspect {
2085 texture_format: wgt::TextureFormat,
2086 requested_aspect: wgt::TextureAspect,
2087 },
2088 #[error(
2089 "Trying to create a view of format {view:?} of a texture with format {texture:?}, \
2090 but this view format is not present in the texture's viewFormat array"
2091 )]
2092 FormatReinterpretation {
2093 texture: wgt::TextureFormat,
2094 view: wgt::TextureFormat,
2095 },
2096 #[error(
2097 "The texture view (`{view:?}`) from transient texture (`{texture:?}`) must have the same usage"
2098 )]
2099 InvalidTransientTextureViewUsage {
2100 texture: wgt::TextureUsages,
2101 view: wgt::TextureUsages,
2102 },
2103 #[error(transparent)]
2104 InvalidResource(#[from] InvalidResourceError),
2105 #[error(transparent)]
2106 MissingFeatures(#[from] MissingFeatures),
2107}
2108
2109impl From<InvalidOrDestroyedResourceError> for CreateTextureViewError {
2110 fn from(value: InvalidOrDestroyedResourceError) -> Self {
2111 match value {
2112 InvalidOrDestroyedResourceError::InvalidResource(e) => Self::InvalidResource(e),
2113 InvalidOrDestroyedResourceError::DestroyedResource(e) => Self::DestroyedResource(e),
2114 }
2115 }
2116}
2117
2118impl WebGpuError for CreateTextureViewError {
2119 fn webgpu_error_type(&self) -> ErrorType {
2120 match self {
2121 Self::Device(e) => e.webgpu_error_type(),
2122
2123 Self::InvalidTextureViewDimension { .. }
2124 | Self::InvalidResource(_)
2125 | Self::InvalidMultisampledTextureViewDimension(_)
2126 | Self::InvalidCubemapTextureDepth { .. }
2127 | Self::InvalidCubemapArrayTextureDepth { .. }
2128 | Self::InvalidCubeTextureViewSize
2129 | Self::ZeroMipLevelCount
2130 | Self::ZeroArrayLayerCount
2131 | Self::TooManyMipLevels { .. }
2132 | Self::TooManyArrayLayers { .. }
2133 | Self::InvalidArrayLayerCount { .. }
2134 | Self::InvalidAspect { .. }
2135 | Self::FormatReinterpretation { .. }
2136 | Self::DestroyedResource(_)
2137 | Self::TextureViewFormatNotRenderable(_)
2138 | Self::TextureViewFormatNotStorage(_)
2139 | Self::InvalidTextureViewUsage { .. }
2140 | Self::InvalidTransientTextureViewUsage { .. }
2141 | Self::MissingFeatures(_) => ErrorType::Validation,
2142 }
2143 }
2144}
2145
2146crate::impl_resource_type!(TextureView);
2147crate::impl_labeled!(TextureView);
2148crate::impl_parent_device!(TextureView);
2149crate::impl_storage_item!(TextureView);
2150
2151pub type ExternalTextureDescriptor<'a> = wgt::ExternalTextureDescriptor<Label<'a>>;
2152
2153#[derive(Debug)]
2154pub struct ExternalTexture {
2155 pub(crate) device: Arc<Device>,
2156 pub(crate) planes: arrayvec::ArrayVec<Arc<TextureView>, 3>,
2158 pub(crate) params: Arc<Buffer>,
2161 pub(crate) label: String,
2163 pub(crate) tracking_data: TrackingData,
2164}
2165
2166impl Drop for ExternalTexture {
2167 fn drop(&mut self) {
2168 resource_log!("Destroy raw {}", self.error_ident());
2169 }
2170}
2171
2172impl ExternalTexture {
2173 pub fn destroy(self: &Arc<Self>) {
2174 self.params.destroy();
2175 }
2176}
2177
2178#[derive(Clone, Debug, Error)]
2179#[non_exhaustive]
2180pub enum CreateExternalTextureError {
2181 #[error(transparent)]
2182 Device(#[from] DeviceError),
2183 #[error(transparent)]
2184 MissingFeatures(#[from] MissingFeatures),
2185 #[error(transparent)]
2186 InvalidResource(#[from] InvalidResourceError),
2187 #[error(transparent)]
2188 CreateBuffer(#[from] CreateBufferError),
2189 #[error(transparent)]
2190 QueueWrite(#[from] queue::QueueWriteError),
2191 #[error("External texture format {format:?} expects {expected} planes, but given {provided}")]
2192 IncorrectPlaneCount {
2193 format: wgt::ExternalTextureFormat,
2194 expected: usize,
2195 provided: usize,
2196 },
2197 #[error("External texture planes cannot be multisampled, but given view with samples = {0}")]
2198 InvalidPlaneMultisample(u32),
2199 #[error("External texture planes expect a filterable float sample type, but given view with format {format:?} (sample type {sample_type:?})")]
2200 InvalidPlaneSampleType {
2201 format: wgt::TextureFormat,
2202 sample_type: wgt::TextureSampleType,
2203 },
2204 #[error("External texture planes expect 2D dimension, but given view with dimension = {0:?}")]
2205 InvalidPlaneDimension(wgt::TextureViewDimension),
2206 #[error(transparent)]
2207 MissingTextureUsage(#[from] MissingTextureUsageError),
2208 #[error("External texture format {format:?} plane {plane} expects format with {expected} components but given view with format {provided:?} ({} components)",
2209 provided.components())]
2210 InvalidPlaneFormat {
2211 format: wgt::ExternalTextureFormat,
2212 plane: usize,
2213 expected: u8,
2214 provided: wgt::TextureFormat,
2215 },
2216}
2217
2218impl WebGpuError for CreateExternalTextureError {
2219 fn webgpu_error_type(&self) -> ErrorType {
2220 match self {
2221 CreateExternalTextureError::Device(e) => e.webgpu_error_type(),
2222 CreateExternalTextureError::MissingFeatures(e) => e.webgpu_error_type(),
2223 CreateExternalTextureError::InvalidResource(e) => e.webgpu_error_type(),
2224 CreateExternalTextureError::CreateBuffer(e) => e.webgpu_error_type(),
2225 CreateExternalTextureError::QueueWrite(e) => e.webgpu_error_type(),
2226 CreateExternalTextureError::MissingTextureUsage(e) => e.webgpu_error_type(),
2227 CreateExternalTextureError::IncorrectPlaneCount { .. }
2228 | CreateExternalTextureError::InvalidPlaneMultisample(_)
2229 | CreateExternalTextureError::InvalidPlaneSampleType { .. }
2230 | CreateExternalTextureError::InvalidPlaneDimension(_)
2231 | CreateExternalTextureError::InvalidPlaneFormat { .. } => ErrorType::Validation,
2232 }
2233 }
2234}
2235
2236crate::impl_resource_type!(ExternalTexture);
2237crate::impl_labeled!(ExternalTexture);
2238crate::impl_parent_device!(ExternalTexture);
2239crate::impl_storage_item!(ExternalTexture);
2240crate::impl_trackable!(ExternalTexture);
2241
2242#[derive(Clone, Debug, PartialEq)]
2244#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
2245pub struct SamplerDescriptor<'a> {
2246 pub label: Label<'a>,
2250 pub address_modes: [wgt::AddressMode; 3],
2252 pub mag_filter: wgt::FilterMode,
2254 pub min_filter: wgt::FilterMode,
2256 pub mipmap_filter: wgt::MipmapFilterMode,
2258 pub lod_min_clamp: f32,
2260 pub lod_max_clamp: f32,
2262 pub compare: Option<wgt::CompareFunction>,
2264 pub anisotropy_clamp: u16,
2266 pub border_color: Option<wgt::SamplerBorderColor>,
2269}
2270
2271#[derive(Debug)]
2272pub struct Sampler {
2273 pub(crate) raw: ManuallyDrop<Box<dyn hal::DynSampler>>,
2274 pub(crate) device: Arc<Device>,
2275 pub(crate) label: String,
2277 pub(crate) tracking_data: TrackingData,
2278 pub(crate) comparison: bool,
2280 pub(crate) filtering: bool,
2282}
2283
2284impl Drop for Sampler {
2285 fn drop(&mut self) {
2286 resource_log!("Destroy raw {}", self.error_ident());
2287 let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
2289 unsafe {
2290 self.device.raw().destroy_sampler(raw);
2291 }
2292 }
2293}
2294
2295impl Sampler {
2296 pub(crate) fn raw(&self) -> &dyn hal::DynSampler {
2297 self.raw.as_ref()
2298 }
2299}
2300
2301#[derive(Copy, Clone)]
2302pub enum SamplerFilterErrorType {
2303 MagFilter,
2304 MinFilter,
2305 MipmapFilter,
2306}
2307
2308impl fmt::Debug for SamplerFilterErrorType {
2309 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2310 match *self {
2311 SamplerFilterErrorType::MagFilter => write!(f, "magFilter"),
2312 SamplerFilterErrorType::MinFilter => write!(f, "minFilter"),
2313 SamplerFilterErrorType::MipmapFilter => write!(f, "mipmapFilter"),
2314 }
2315 }
2316}
2317
2318#[derive(Clone, Debug, Error)]
2319#[non_exhaustive]
2320pub enum CreateSamplerError {
2321 #[error(transparent)]
2322 Device(#[from] DeviceError),
2323 #[error("Invalid lodMinClamp: {0}. Must be greater or equal to 0.0")]
2324 InvalidLodMinClamp(f32),
2325 #[error("Invalid lodMaxClamp: {lod_max_clamp}. Must be greater or equal to lodMinClamp (which is {lod_min_clamp}).")]
2326 InvalidLodMaxClamp {
2327 lod_min_clamp: f32,
2328 lod_max_clamp: f32,
2329 },
2330 #[error("Invalid anisotropic clamp: {0}. Must be at least 1.")]
2331 InvalidAnisotropy(u16),
2332 #[error("Invalid filter mode for {filter_type:?}: {filter_mode:?}. When anistropic clamp is not 1 (it is {anisotropic_clamp}), all filter modes must be linear.")]
2333 InvalidFilterModeWithAnisotropy {
2334 filter_type: SamplerFilterErrorType,
2335 filter_mode: wgt::FilterMode,
2336 anisotropic_clamp: u16,
2337 },
2338 #[error("Invalid filter mode for {filter_type:?}: {filter_mode:?}. When anistropic clamp is not 1 (it is {anisotropic_clamp}), all filter modes must be linear.")]
2339 InvalidMipmapFilterModeWithAnisotropy {
2340 filter_type: SamplerFilterErrorType,
2341 filter_mode: wgt::MipmapFilterMode,
2342 anisotropic_clamp: u16,
2343 },
2344 #[error(transparent)]
2345 MissingFeatures(#[from] MissingFeatures),
2346}
2347
2348crate::impl_resource_type!(Sampler);
2349crate::impl_labeled!(Sampler);
2350crate::impl_parent_device!(Sampler);
2351crate::impl_storage_item!(Sampler);
2352crate::impl_trackable!(Sampler);
2353
2354impl WebGpuError for CreateSamplerError {
2355 fn webgpu_error_type(&self) -> ErrorType {
2356 match self {
2357 Self::Device(e) => e.webgpu_error_type(),
2358 Self::MissingFeatures(e) => e.webgpu_error_type(),
2359
2360 Self::InvalidLodMinClamp(_)
2361 | Self::InvalidLodMaxClamp { .. }
2362 | Self::InvalidAnisotropy(_)
2363 | Self::InvalidFilterModeWithAnisotropy { .. }
2364 | Self::InvalidMipmapFilterModeWithAnisotropy { .. } => ErrorType::Validation,
2365 }
2366 }
2367}
2368
2369#[derive(Clone, Debug, Error)]
2370#[non_exhaustive]
2371pub enum CreateQuerySetError {
2372 #[error(transparent)]
2373 Device(#[from] DeviceError),
2374 #[error("QuerySets cannot be made with zero queries")]
2375 ZeroCount,
2376 #[error("{count} is too many queries for a single QuerySet. QuerySets cannot be made more than {maximum} queries.")]
2377 TooManyQueries { count: u32, maximum: u32 },
2378 #[error(transparent)]
2379 MissingFeatures(#[from] MissingFeatures),
2380}
2381
2382impl WebGpuError for CreateQuerySetError {
2383 fn webgpu_error_type(&self) -> ErrorType {
2384 match self {
2385 Self::Device(e) => e.webgpu_error_type(),
2386 Self::MissingFeatures(e) => e.webgpu_error_type(),
2387
2388 Self::TooManyQueries { .. } | Self::ZeroCount => ErrorType::Validation,
2389 }
2390 }
2391}
2392
2393pub type QuerySetDescriptor<'a> = wgt::QuerySetDescriptor<Label<'a>>;
2394
2395#[derive(Debug)]
2396pub struct QuerySet {
2397 pub(crate) raw: Snatchable<Box<dyn hal::DynQuerySet>>,
2398 pub(crate) device: Arc<Device>,
2399 pub(crate) label: String,
2401 pub(crate) tracking_data: TrackingData,
2402 pub(crate) desc: wgt::QuerySetDescriptor<()>,
2403 pub(crate) initialized_slots: Mutex<bit_vec::BitVec>,
2404}
2405
2406impl RawResourceAccess for QuerySet {
2407 type DynResource = dyn hal::DynQuerySet;
2408
2409 fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2410 self.raw.get(guard).map(|b| b.as_ref())
2411 }
2412}
2413
2414impl QuerySet {
2415 pub fn destroy(self: &Arc<Self>) {
2416 let device = &self.device;
2417
2418 let temp = {
2419 let mut snatch_guard = self.device.snatchable_lock.write();
2420
2421 let raw = match self.raw.snatch(&mut snatch_guard) {
2422 Some(raw) => raw,
2423 None => {
2424 return;
2426 }
2427 };
2428
2429 drop(snatch_guard);
2430
2431 queue::TempResource::DestroyedQuerySet(DestroyedQuerySet {
2432 raw: ManuallyDrop::new(raw),
2433 device: Arc::clone(&self.device),
2434 label: self.label().to_owned(),
2435 })
2436 };
2437
2438 let Some(queue) = device.get_queue() else {
2439 return;
2440 };
2441
2442 let mut life_lock = queue.lock_life();
2443 let last_submit_index = life_lock.get_query_set_latest_submission_index(self);
2444 if let Some(last_submit_index) = last_submit_index {
2445 life_lock.schedule_resource_destruction(temp, last_submit_index);
2446 }
2447 }
2448}
2449
2450impl Drop for QuerySet {
2451 fn drop(&mut self) {
2452 resource_log!("Destroy raw {}", self.error_ident());
2453 if let Some(raw) = self.raw.take() {
2454 unsafe {
2456 self.device.raw().destroy_query_set(raw);
2457 }
2458 }
2459 }
2460}
2461
2462crate::impl_resource_type!(QuerySet);
2463crate::impl_labeled!(QuerySet);
2464crate::impl_parent_device!(QuerySet);
2465crate::impl_storage_item!(QuerySet);
2466crate::impl_trackable!(QuerySet);
2467
2468#[derive(Debug)]
2470pub struct DestroyedQuerySet {
2471 raw: ManuallyDrop<Box<dyn hal::DynQuerySet>>,
2472 device: Arc<Device>,
2473 label: String,
2474}
2475
2476impl DestroyedQuerySet {
2477 pub fn label(&self) -> &dyn fmt::Debug {
2478 &self.label
2479 }
2480}
2481
2482impl Drop for DestroyedQuerySet {
2483 fn drop(&mut self) {
2484 resource_log!("Destroy raw QuerySet (destroyed) {:?}", self.label());
2485 let raw = unsafe { ManuallyDrop::take(&mut self.raw) };
2487 unsafe {
2488 hal::DynDevice::destroy_query_set(self.device.raw(), raw);
2489 }
2490 }
2491}
2492
2493pub type BlasDescriptor<'a> = wgt::CreateBlasDescriptor<Label<'a>>;
2494pub type TlasDescriptor<'a> = wgt::CreateTlasDescriptor<Label<'a>>;
2495
2496pub type BlasPrepareCompactResult = Result<(), BlasPrepareCompactError>;
2497
2498#[cfg(send_sync)]
2499pub type BlasCompactCallback = Box<dyn FnOnce(BlasPrepareCompactResult) + Send + 'static>;
2500#[cfg(not(send_sync))]
2501pub type BlasCompactCallback = Box<dyn FnOnce(BlasPrepareCompactResult) + 'static>;
2502
2503pub(crate) struct BlasPendingCompact {
2504 pub(crate) op: Option<BlasCompactCallback>,
2505 pub(crate) _parent_blas: Arc<Blas>,
2507}
2508
2509impl fmt::Debug for BlasPendingCompact {
2510 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2511 f.debug_struct("BlasPendingCompact")
2512 .field("op", &())
2513 .field("_parent_blas", &self._parent_blas)
2514 .finish()
2515 }
2516}
2517
2518#[derive(Debug)]
2519pub(crate) enum BlasCompactState {
2520 Compacted,
2522 Waiting(BlasPendingCompact),
2524 Ready { size: wgt::BufferAddress },
2526 Idle,
2528}
2529
2530#[cfg(send_sync)]
2531unsafe impl Send for BlasCompactState {}
2532#[cfg(send_sync)]
2533unsafe impl Sync for BlasCompactState {}
2534
2535#[derive(Debug)]
2536pub struct Blas {
2537 pub(crate) raw: Snatchable<Box<dyn hal::DynAccelerationStructure>>,
2538 pub(crate) device: Arc<Device>,
2539 pub(crate) size_info: hal::AccelerationStructureBuildSizes,
2540 pub(crate) sizes: wgt::BlasGeometrySizeDescriptors,
2541 pub(crate) flags: wgt::AccelerationStructureFlags,
2542 pub(crate) update_mode: wgt::AccelerationStructureUpdateMode,
2543 pub(crate) built_index: RwLock<Option<NonZeroU64>>,
2544 pub(crate) handle: u64,
2545 pub(crate) label: String,
2547 pub(crate) tracking_data: TrackingData,
2548 pub(crate) compaction_buffer: Option<ManuallyDrop<Box<dyn hal::DynBuffer>>>,
2549 pub(crate) compacted_state: Mutex<BlasCompactState>,
2550}
2551
2552impl Drop for Blas {
2553 fn drop(&mut self) {
2554 resource_log!("Destroy raw {}", self.error_ident());
2555 if let Some(raw) = self.raw.take() {
2557 unsafe {
2558 self.device.raw().destroy_acceleration_structure(raw);
2559 }
2560 }
2561 if let Some(mut raw) = self.compaction_buffer.take() {
2562 unsafe {
2563 self.device
2564 .raw()
2565 .destroy_buffer(ManuallyDrop::take(&mut raw))
2566 }
2567 }
2568 }
2569}
2570
2571impl RawResourceAccess for Blas {
2572 type DynResource = dyn hal::DynAccelerationStructure;
2573
2574 fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2575 self.raw.get(guard).map(|it| it.as_ref())
2576 }
2577}
2578
2579impl Blas {
2580 pub(crate) fn prepare_compact_async(
2581 self: &Arc<Self>,
2582 op: Option<BlasCompactCallback>,
2583 ) -> Result<SubmissionIndex, (Option<BlasCompactCallback>, BlasPrepareCompactError)> {
2584 let device = &self.device;
2585 if let Err(e) = device.check_is_valid() {
2586 return Err((op, e.into()));
2587 }
2588
2589 if self.built_index.read().is_none() {
2590 return Err((op, BlasPrepareCompactError::NotBuilt));
2591 }
2592
2593 if !self
2594 .flags
2595 .contains(wgt::AccelerationStructureFlags::ALLOW_COMPACTION)
2596 {
2597 return Err((op, BlasPrepareCompactError::CompactionUnsupported));
2598 }
2599
2600 let mut state = self.compacted_state.lock();
2601 *state = match *state {
2602 BlasCompactState::Compacted => {
2603 return Err((op, BlasPrepareCompactError::DoubleCompaction))
2604 }
2605 BlasCompactState::Waiting(_) => {
2606 return Err((op, BlasPrepareCompactError::CompactionPreparingAlready))
2607 }
2608 BlasCompactState::Ready { .. } => {
2609 return Err((op, BlasPrepareCompactError::CompactionPreparingAlready))
2610 }
2611 BlasCompactState::Idle => BlasCompactState::Waiting(BlasPendingCompact {
2612 op,
2613 _parent_blas: self.clone(),
2614 }),
2615 };
2616
2617 let submit_index = if let Some(queue) = device.get_queue() {
2618 queue.lock_life().prepare_compact(self).unwrap_or(0) } else {
2620 let (mut callback, status) = self.read_back_compact_size().unwrap();
2622 if let Some(callback) = callback.take() {
2623 callback(status);
2624 }
2625 0
2626 };
2627
2628 Ok(submit_index)
2629 }
2630
2631 #[must_use]
2633 pub(crate) fn read_back_compact_size(&self) -> Option<BlasCompactReadyPendingClosure> {
2634 let mut state = self.compacted_state.lock();
2635 let pending_compact = match mem::replace(&mut *state, BlasCompactState::Idle) {
2636 BlasCompactState::Waiting(pending_mapping) => pending_mapping,
2637 BlasCompactState::Idle => return None,
2639 BlasCompactState::Ready { .. } => {
2640 unreachable!("This should be validated out by `prepare_for_compaction`")
2641 }
2642 _ => panic!("No pending mapping."),
2643 };
2644 let status = {
2645 let compaction_buffer = self.compaction_buffer.as_ref().unwrap().as_ref();
2646 unsafe {
2647 let map_res = self.device.raw().map_buffer(
2648 compaction_buffer,
2649 0..size_of::<wgpu_types::BufferAddress>() as wgt::BufferAddress,
2650 );
2651 match map_res {
2652 Ok(mapping) => {
2653 if !mapping.is_coherent {
2654 #[expect(clippy::single_range_in_vec_init, reason = "intentional")]
2655 self.device.raw().invalidate_mapped_ranges(
2656 compaction_buffer,
2657 &[0..size_of::<wgpu_types::BufferAddress>() as wgt::BufferAddress],
2658 );
2659 }
2660 let size = core::ptr::read_unaligned(
2661 mapping.ptr.as_ptr().cast::<wgt::BufferAddress>(),
2662 );
2663 self.device.raw().unmap_buffer(compaction_buffer);
2664 if self.size_info.acceleration_structure_size != 0 {
2665 debug_assert_ne!(size, 0);
2666 }
2667 *state = BlasCompactState::Ready { size };
2668 Ok(())
2669 }
2670 Err(err) => Err(BlasPrepareCompactError::from(DeviceError::from_hal(err))),
2671 }
2672 }
2673 };
2674 Some((pending_compact.op, status))
2675 }
2676}
2677
2678crate::impl_resource_type!(Blas);
2679crate::impl_labeled!(Blas);
2680crate::impl_parent_device!(Blas);
2681crate::impl_storage_item!(Blas);
2682crate::impl_trackable!(Blas);
2683
2684#[derive(Debug)]
2685pub struct Tlas {
2686 pub(crate) raw: Snatchable<Box<dyn hal::DynAccelerationStructure>>,
2687 pub(crate) device: Arc<Device>,
2688 pub(crate) size_info: hal::AccelerationStructureBuildSizes,
2689 pub(crate) max_instance_count: u32,
2690 pub(crate) flags: wgt::AccelerationStructureFlags,
2691 pub(crate) update_mode: wgt::AccelerationStructureUpdateMode,
2692 pub(crate) built_index: RwLock<Option<NonZeroU64>>,
2693 pub(crate) dependencies: RwLock<Vec<Arc<Blas>>>,
2694 pub(crate) instance_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
2695 pub(crate) label: String,
2697 pub(crate) tracking_data: TrackingData,
2698}
2699
2700impl Drop for Tlas {
2701 fn drop(&mut self) {
2702 unsafe {
2703 resource_log!("Destroy raw {}", self.error_ident());
2704 if let Some(structure) = self.raw.take() {
2705 self.device.raw().destroy_acceleration_structure(structure);
2706 }
2707 let buffer = ManuallyDrop::take(&mut self.instance_buffer);
2708 self.device.raw().destroy_buffer(buffer);
2709 }
2710 }
2711}
2712
2713impl RawResourceAccess for Tlas {
2714 type DynResource = dyn hal::DynAccelerationStructure;
2715
2716 fn raw<'a>(&'a self, guard: &'a SnatchGuard) -> Option<&'a Self::DynResource> {
2717 self.raw.get(guard).map(|raw| raw.as_ref())
2718 }
2719}
2720
2721crate::impl_resource_type!(Tlas);
2722crate::impl_labeled!(Tlas);
2723crate::impl_parent_device!(Tlas);
2724crate::impl_storage_item!(Tlas);
2725crate::impl_trackable!(Tlas);