1use alloc::{
2 borrow::Cow,
3 boxed::Box,
4 string::{String, ToString as _},
5 sync::{Arc, Weak},
6 vec::Vec,
7};
8use core::{
9 fmt,
10 mem::{self, ManuallyDrop},
11 num::NonZeroU32,
12 sync::atomic::{AtomicBool, Ordering},
13};
14use hal::ShouldBeNonZeroExt;
15
16use arrayvec::ArrayVec;
17use bitflags::Flags;
18use smallvec::SmallVec;
19use wgt::{
20 math::align_to, DeviceLostReason, TextureFormat, TextureSampleType, TextureSelector,
21 TextureViewDimension,
22};
23
24#[cfg(feature = "trace")]
25use crate::device::trace;
26use crate::{
27 api_log,
28 binding_model::{
29 self, BindGroup, BindGroupLateBufferBindingInfo, BindGroupLayout,
30 BindGroupLayoutEntryError, BindGroupLayoutState, CreateBindGroupError,
31 CreateBindGroupLayoutError,
32 },
33 command, conv,
34 device::{
35 bgl, create_validator, features_to_naga_capabilities, life::WaitIdleError, map_buffer,
36 AttachmentData, DeviceLostInvocation, HostMap, MissingDownlevelFlags, MissingFeatures,
37 RenderPassContext,
38 },
39 hal_label,
40 init_tracker::{
41 BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange,
42 TextureInitTrackerAction,
43 },
44 instance::{Adapter, RequestDeviceError},
45 lock::{rank, Mutex, RwLock},
46 pipeline::{self, ColorStateError},
47 pool::ResourcePool,
48 present,
49 resource::{
50 self, Buffer, ExternalTexture, Fallible, Labeled, ParentDevice, QuerySet,
51 RawResourceAccess, ResourceState, Sampler, StagingBuffer, Texture, TextureView,
52 TextureViewNotRenderableReason, Tlas, TrackingData,
53 },
54 resource_log,
55 snatch::{SnatchGuard, SnatchLock, Snatchable},
56 timestamp_normalization::TIMESTAMP_NORMALIZATION_BUFFER_USES,
57 track::{BindGroupStates, DeviceTracker, TrackerIndexAllocators, UsageScope, UsageScopePool},
58 validation::{self, check_color_attachment_count, PassthroughInterface, ShaderMetaData},
59 weak_vec::WeakVec,
60 FastHashMap, LabelHelpers, OnceCellOrLock,
61};
62
63use super::{
64 queue::Queue, surface_config::validate_surface_configuration, DeviceDescriptor, DeviceError,
65 DeviceLostClosure, UserClosures, ENTRYPOINT_FAILURE_ERROR, ZERO_BUFFER_SIZE,
66};
67
68#[cfg(supports_64bit_atomics)]
69use core::sync::atomic::AtomicU64;
70#[cfg(not(supports_64bit_atomics))]
71use portable_atomic::AtomicU64;
72
73pub(crate) struct CommandIndices {
74 pub(crate) active_submission_index: hal::FenceValue,
82 pub(crate) next_acceleration_structure_build_command_index: u64,
83}
84
85#[repr(C)]
92#[derive(Copy, Clone, bytemuck::Zeroable, bytemuck::Pod)]
93pub struct ExternalTextureParams {
94 pub yuv_conversion_matrix: [f32; 16],
99
100 pub gamut_conversion_matrix: [f32; 12],
113
114 pub src_transfer_function: wgt::ExternalTextureTransferFunction,
118
119 pub dst_transfer_function: wgt::ExternalTextureTransferFunction,
122
123 pub sample_transform: [f32; 6],
136
137 pub load_transform: [f32; 6],
153
154 pub size: [u32; 2],
167
168 pub num_planes: u32,
172 pub _padding: [u8; 4],
174}
175
176impl ExternalTextureParams {
177 pub fn from_desc<L>(desc: &wgt::ExternalTextureDescriptor<L>) -> Self {
178 let gamut_conversion_matrix = [
179 desc.gamut_conversion_matrix[0],
180 desc.gamut_conversion_matrix[1],
181 desc.gamut_conversion_matrix[2],
182 0.0, desc.gamut_conversion_matrix[3],
184 desc.gamut_conversion_matrix[4],
185 desc.gamut_conversion_matrix[5],
186 0.0, desc.gamut_conversion_matrix[6],
188 desc.gamut_conversion_matrix[7],
189 desc.gamut_conversion_matrix[8],
190 0.0, ];
192
193 Self {
194 yuv_conversion_matrix: desc.yuv_conversion_matrix,
195 gamut_conversion_matrix,
196 src_transfer_function: desc.src_transfer_function,
197 dst_transfer_function: desc.dst_transfer_function,
198 size: [desc.width, desc.height],
199 sample_transform: desc.sample_transform,
200 load_transform: desc.load_transform,
201 num_planes: desc.num_planes() as u32,
202 _padding: Default::default(),
203 }
204 }
205}
206
207pub struct Device {
210 raw: Box<dyn hal::DynDevice>,
211 pub(crate) adapter: Arc<Adapter>,
212 pub(crate) queue: OnceCellOrLock<Weak<Queue>>,
213 pub(crate) zero_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
214 pub(crate) empty_bgl: ManuallyDrop<Box<dyn hal::DynBindGroupLayout>>,
215 label: String,
217
218 pub(crate) command_allocator: command::CommandAllocator,
219
220 pub(crate) command_indices: RwLock<CommandIndices>,
221
222 pub(crate) last_successful_submission_index: hal::AtomicFenceValue,
232
233 pub(crate) fence: ManuallyDrop<Box<dyn hal::DynFence>>,
234 pub(crate) snatchable_lock: SnatchLock,
235
236 pub(crate) valid: AtomicBool,
248
249 pub(crate) device_lost_closure: Mutex<Option<DeviceLostClosure>>,
253
254 pub(crate) trackers: Mutex<DeviceTracker>,
256 pub(crate) tracker_indices: TrackerIndexAllocators,
257 pub(crate) bgl_pool: ResourcePool<bgl::EntryMap, BindGroupLayout>,
259 pub(crate) alignments: hal::Alignments,
260 pub(crate) limits: wgt::Limits,
261 pub(crate) features: wgt::Features,
262 pub(crate) downlevel: wgt::DownlevelCapabilities,
263 pub(crate) ordered_buffer_usages: wgt::BufferUses,
268 pub(crate) ordered_texture_usages: wgt::TextureUses,
273 pub(crate) instance_flags: wgt::InstanceFlags,
274 pub(crate) deferred_destroy: Mutex<Vec<DeferredDestroy>>,
275 pub(crate) usage_scopes: UsageScopePool,
276 pub(crate) indirect_validation: Option<crate::indirect_validation::IndirectValidation>,
277 pub(crate) timestamp_normalizer:
279 OnceCellOrLock<crate::timestamp_normalization::TimestampNormalizer>,
280 pub(crate) default_external_texture_params_buffer: ManuallyDrop<Box<dyn hal::DynBuffer>>,
285 #[cfg(feature = "trace")]
287 pub(crate) trace: Mutex<Option<Box<dyn trace::Trace + Send + Sync + 'static>>>,
288}
289
290pub(crate) enum DeferredDestroy {
291 TextureViews(WeakVec<TextureView>),
292 BindGroups(WeakVec<BindGroup>),
293}
294
295impl fmt::Debug for Device {
296 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
297 f.debug_struct("Device")
298 .field("label", &self.label())
299 .field("limits", &self.limits)
300 .field("features", &self.features)
301 .field("downlevel", &self.downlevel)
302 .finish()
303 }
304}
305
306impl Drop for Device {
307 fn drop(&mut self) {
308 resource_log!("Drop {}", self.error_ident());
309
310 let zero_buffer = unsafe { ManuallyDrop::take(&mut self.zero_buffer) };
313 let empty_bgl = unsafe { ManuallyDrop::take(&mut self.empty_bgl) };
315 let default_external_texture_params_buffer =
318 unsafe { ManuallyDrop::take(&mut self.default_external_texture_params_buffer) };
319 let fence = unsafe { ManuallyDrop::take(&mut self.fence) };
321 if let Some(indirect_validation) = self.indirect_validation.take() {
322 indirect_validation.dispose(self.raw.as_ref());
323 }
324 if let Some(timestamp_normalizer) = self.timestamp_normalizer.take() {
325 timestamp_normalizer.dispose(self.raw.as_ref());
326 }
327 unsafe {
328 self.raw.destroy_buffer(zero_buffer);
329 self.raw.destroy_bind_group_layout(empty_bgl);
330 self.raw
331 .destroy_buffer(default_external_texture_params_buffer);
332 self.raw.destroy_fence(fence);
333 }
334 }
335}
336
337impl Device {
338 pub(crate) fn raw(&self) -> &dyn hal::DynDevice {
339 self.raw.as_ref()
340 }
341 pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> {
342 if self.features.contains(feature) {
343 Ok(())
344 } else {
345 Err(MissingFeatures(feature))
346 }
347 }
348
349 pub(crate) fn require_downlevel_flags(
350 &self,
351 flags: wgt::DownlevelFlags,
352 ) -> Result<(), MissingDownlevelFlags> {
353 if self.downlevel.flags.contains(flags) {
354 Ok(())
355 } else {
356 Err(MissingDownlevelFlags(flags))
357 }
358 }
359
360 pub unsafe fn start_graphics_debugger_capture(&self) {
366 api_log!("Device::start_graphics_debugger_capture");
367
368 if !self.is_valid() {
369 return;
370 }
371 unsafe { self.raw().start_graphics_debugger_capture() };
372 }
373
374 pub unsafe fn stop_graphics_debugger_capture(&self) {
380 api_log!("Device::stop_graphics_debugger_capture");
381
382 if !self.is_valid() {
383 return;
384 }
385 unsafe { self.raw().stop_graphics_debugger_capture() };
386 }
387}
388
389impl Device {
390 pub(crate) fn new(
391 raw_device: Box<dyn hal::DynDevice>,
392 adapter: &Arc<Adapter>,
393 desc: &DeviceDescriptor,
394 instance_flags: wgt::InstanceFlags,
395 ) -> Result<Self, DeviceError> {
396 #[cfg(not(feature = "trace"))]
397 match &desc.trace {
398 wgt::Trace::Off => {}
399 _ => {
400 log::error!("wgpu-core feature 'trace' is not enabled");
401 }
402 };
403 #[cfg(feature = "trace")]
404 let trace: Option<Box<dyn trace::Trace + Send + Sync + 'static>> = match &desc.trace {
405 wgt::Trace::Off => None,
406 wgt::Trace::Directory(dir) => match trace::DiskTrace::new(dir.clone()) {
407 Ok(mut trace) => {
408 trace::Trace::add(
409 &mut trace,
410 trace::Action::Init {
411 desc: wgt::DeviceDescriptor {
412 trace: wgt::Trace::Off,
413 ..desc.clone()
414 },
415 backend: adapter.backend(),
416 },
417 );
418 Some(Box::new(trace))
419 }
420 Err(e) => {
421 log::error!("Unable to start a trace in '{dir:?}': {e}");
422 None
423 }
424 },
425 wgt::Trace::Memory => {
426 let mut trace = trace::MemoryTrace::new();
427 trace::Trace::add(
428 &mut trace,
429 trace::Action::Init {
430 desc: wgt::DeviceDescriptor {
431 trace: wgt::Trace::Off,
432 ..desc.clone()
433 },
434 backend: adapter.backend(),
435 },
436 );
437 Some(Box::new(trace))
438 }
439 t => {
442 log::error!("unimplemented wgpu_types::Trace variant {t:?}");
443 None
444 }
445 };
446
447 let ordered_buffer_usages = adapter.raw.adapter.get_ordered_buffer_usages();
448 let ordered_texture_usages = adapter.raw.adapter.get_ordered_texture_usages();
449
450 let fence = unsafe { raw_device.create_fence() }.map_err(DeviceError::from_hal)?;
451
452 let command_allocator = command::CommandAllocator::new();
453
454 let rt_uses = if desc
455 .required_features
456 .intersects(wgt::Features::EXPERIMENTAL_RAY_QUERY)
457 {
458 wgt::BufferUses::TOP_LEVEL_ACCELERATION_STRUCTURE_INPUT
459 } else {
460 wgt::BufferUses::empty()
461 };
462
463 let zero_buffer = unsafe {
465 raw_device.create_buffer(&hal::BufferDescriptor {
466 label: hal_label(Some("(wgpu internal) zero init buffer"), instance_flags),
467 size: ZERO_BUFFER_SIZE,
468 usage: wgt::BufferUses::COPY_SRC | wgt::BufferUses::COPY_DST | rt_uses,
469 memory_flags: hal::MemoryFlags::empty(),
470 })
471 }
472 .map_err(DeviceError::from_hal)?;
473
474 let empty_bgl = unsafe {
475 raw_device.create_bind_group_layout(&hal::BindGroupLayoutDescriptor {
476 label: None,
477 flags: hal::BindGroupLayoutFlags::empty(),
478 entries: &[],
479 })
480 }
481 .map_err(DeviceError::from_hal)?;
482
483 let default_external_texture_params_buffer = unsafe {
484 raw_device.create_buffer(&hal::BufferDescriptor {
485 label: hal_label(
486 Some("(wgpu internal) default external texture params buffer"),
487 instance_flags,
488 ),
489 size: size_of::<ExternalTextureParams>() as _,
490 usage: wgt::BufferUses::COPY_DST | wgt::BufferUses::UNIFORM,
491 memory_flags: hal::MemoryFlags::empty(),
492 })
493 }
494 .map_err(DeviceError::from_hal)?;
495
496 let alignments = adapter.raw.capabilities.alignments.clone();
498 let downlevel = adapter.raw.capabilities.downlevel.clone();
499 let limits = &adapter.raw.capabilities.limits;
500
501 let enable_indirect_validation = instance_flags
502 .contains(wgt::InstanceFlags::VALIDATION_INDIRECT_CALL)
503 && downlevel.flags.contains(
504 wgt::DownlevelFlags::INDIRECT_EXECUTION | wgt::DownlevelFlags::COMPUTE_SHADERS,
505 )
506 && limits.max_storage_buffers_per_shader_stage >= 2;
507
508 let indirect_validation = if enable_indirect_validation {
509 Some(crate::indirect_validation::IndirectValidation::new(
510 raw_device.as_ref(),
511 &desc.required_limits,
512 &desc.required_features,
513 instance_flags,
514 adapter.backend(),
515 )?)
516 } else {
517 None
518 };
519
520 Ok(Self {
521 raw: raw_device,
522 adapter: adapter.clone(),
523 queue: OnceCellOrLock::new(),
524 zero_buffer: ManuallyDrop::new(zero_buffer),
525 empty_bgl: ManuallyDrop::new(empty_bgl),
526 default_external_texture_params_buffer: ManuallyDrop::new(
527 default_external_texture_params_buffer,
528 ),
529 label: desc.label.to_string(),
530 command_allocator,
531 command_indices: RwLock::new(
532 rank::DEVICE_COMMAND_INDICES,
533 CommandIndices {
534 active_submission_index: 0,
535 next_acceleration_structure_build_command_index: 1,
537 },
538 ),
539 last_successful_submission_index: AtomicU64::new(0),
540 fence: ManuallyDrop::new(fence),
541 snatchable_lock: unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) },
542 valid: AtomicBool::new(true),
543 device_lost_closure: Mutex::new(rank::DEVICE_LOST_CLOSURE, None),
544 trackers: Mutex::new(
545 rank::DEVICE_TRACKERS,
546 DeviceTracker::new(ordered_buffer_usages, ordered_texture_usages),
547 ),
548 tracker_indices: TrackerIndexAllocators::new(),
549 bgl_pool: ResourcePool::new(),
550 #[cfg(feature = "trace")]
551 trace: Mutex::new(rank::DEVICE_TRACE, trace),
552 alignments,
553 limits: desc.required_limits.clone(),
554 features: desc.required_features,
555 downlevel,
556 ordered_buffer_usages,
557 ordered_texture_usages,
558 instance_flags,
559 deferred_destroy: Mutex::new(rank::DEVICE_DEFERRED_DESTROY, Vec::new()),
560 usage_scopes: Mutex::new(rank::DEVICE_USAGE_SCOPES, Default::default()),
561 timestamp_normalizer: OnceCellOrLock::new(),
562 indirect_validation,
563 })
564 }
565
566 fn init_default_external_texture_params_buffer(self: &Arc<Self>) -> Result<(), DeviceError> {
571 let data = ExternalTextureParams {
572 #[rustfmt::skip]
573 yuv_conversion_matrix: [
574 1.0, 0.0, 0.0, 0.0,
575 0.0, 1.0, 0.0, 0.0,
576 0.0, 0.0, 1.0, 0.0,
577 0.0, 0.0, 0.0, 1.0,
578 ],
579 #[rustfmt::skip]
580 gamut_conversion_matrix: [
581 1.0, 0.0, 0.0, 0.0,
582 0.0, 1.0, 0.0, 0.0,
583 0.0, 0.0, 1.0, 0.0,
584 ],
585 src_transfer_function: Default::default(),
586 dst_transfer_function: Default::default(),
587 size: [0, 0],
588 #[rustfmt::skip]
589 sample_transform: [
590 1.0, 0.0,
591 0.0, 1.0,
592 0.0, 0.0
593 ],
594 #[rustfmt::skip]
595 load_transform: [
596 1.0, 0.0,
597 0.0, 1.0,
598 0.0, 0.0
599 ],
600 num_planes: 1,
601 _padding: Default::default(),
602 };
603 let mut staging_buffer =
604 StagingBuffer::new(self, wgt::BufferSize::new(size_of_val(&data) as _).unwrap())?;
605 staging_buffer.write(bytemuck::bytes_of(&data));
606 let staging_buffer = staging_buffer.flush();
607
608 let params_buffer = self.default_external_texture_params_buffer.as_ref();
609 let queue = self.get_queue().unwrap();
610 let mut pending_writes = queue.pending_writes.lock();
611
612 unsafe {
613 pending_writes
614 .command_encoder
615 .transition_buffers(&[hal::BufferBarrier {
616 buffer: params_buffer,
617 usage: hal::StateTransition {
618 from: wgt::BufferUses::MAP_WRITE,
619 to: wgt::BufferUses::COPY_DST,
620 },
621 }]);
622 pending_writes.command_encoder.copy_buffer_to_buffer(
623 staging_buffer.raw(),
624 params_buffer,
625 &[hal::BufferCopy {
626 src_offset: 0,
627 dst_offset: 0,
628 size: staging_buffer.size,
629 }],
630 );
631 pending_writes.consume(staging_buffer);
632 pending_writes
633 .command_encoder
634 .transition_buffers(&[hal::BufferBarrier {
635 buffer: params_buffer,
636 usage: hal::StateTransition {
637 from: wgt::BufferUses::COPY_DST,
638 to: wgt::BufferUses::UNIFORM,
639 },
640 }]);
641 }
642
643 Ok(())
644 }
645
646 pub fn late_init_resources_with_queue(self: &Arc<Self>) -> Result<(), RequestDeviceError> {
647 let queue = self.get_queue().unwrap();
648
649 let timestamp_normalizer = crate::timestamp_normalization::TimestampNormalizer::new(
650 self,
651 queue.get_timestamp_period(),
652 )?;
653
654 self.timestamp_normalizer
655 .set(timestamp_normalizer)
656 .unwrap_or_else(|_| panic!("Called late_init_resources_with_queue twice"));
657
658 self.init_default_external_texture_params_buffer()?;
659
660 Ok(())
661 }
662
663 pub fn backend(&self) -> wgt::Backend {
665 self.adapter.backend()
666 }
667
668 pub fn is_valid(&self) -> bool {
669 self.valid.load(Ordering::Acquire)
670 }
671
672 pub fn check_is_valid(&self) -> Result<(), DeviceError> {
673 if self.is_valid() {
674 Ok(())
675 } else {
676 Err(DeviceError::Lost)
677 }
678 }
679
680 #[cfg(feature = "trace")]
684 pub fn take_trace(&self) -> Option<Box<dyn trace::Trace + Send + Sync + 'static>> {
685 self.trace.lock().take()
686 }
687
688 pub fn lose_if_oom(&self) {
695 let _ = self
696 .raw()
697 .check_if_oom()
698 .map_err(|e| self.handle_hal_error(e));
699 }
700
701 pub fn handle_hal_error(&self, error: hal::DeviceError) -> DeviceError {
702 match error {
703 hal::DeviceError::OutOfMemory
704 | hal::DeviceError::Lost
705 | hal::DeviceError::Unexpected => {
706 self.lose(&error.to_string());
707 }
708 }
709 DeviceError::from_hal(error)
710 }
711
712 pub fn handle_hal_error_with_nonfatal_oom(&self, error: hal::DeviceError) -> DeviceError {
713 match error {
714 hal::DeviceError::OutOfMemory => DeviceError::from_hal(error),
715 error => self.handle_hal_error(error),
716 }
717 }
718
719 pub(crate) fn deferred_resource_destruction(&self) {
727 let deferred_destroy = mem::take(&mut *self.deferred_destroy.lock());
729 for item in deferred_destroy {
730 match item {
731 DeferredDestroy::TextureViews(views) => {
732 for view in views {
733 let Some(view) = view.upgrade() else {
734 continue;
735 };
736 let Some(raw_view) = view.raw.snatch(&mut self.snatchable_lock.write())
737 else {
738 continue;
739 };
740
741 resource_log!("Destroy raw {}", view.error_ident());
742
743 unsafe {
744 self.raw().destroy_texture_view(raw_view);
745 }
746 }
747 }
748 DeferredDestroy::BindGroups(bind_groups) => {
749 for bind_group in bind_groups {
750 let Some(bind_group) = bind_group.upgrade() else {
751 continue;
752 };
753 let Some(raw_bind_group) =
754 bind_group.raw.snatch(&mut self.snatchable_lock.write())
755 else {
756 continue;
757 };
758
759 resource_log!("Destroy raw {}", bind_group.error_ident());
760
761 unsafe {
762 self.raw().destroy_bind_group(raw_bind_group);
763 }
764 }
765 }
766 }
767 }
768 }
769
770 pub fn get_queue(&self) -> Option<Arc<Queue>> {
771 self.queue.get().as_ref()?.upgrade()
772 }
773
774 pub fn set_queue(&self, queue: &Arc<Queue>) {
775 assert!(self.queue.set(Arc::downgrade(queue)).is_ok());
776 }
777
778 pub fn poll(
779 &self,
780 poll_type: wgt::PollType<crate::SubmissionIndex>,
781 ) -> Result<wgt::PollStatus, WaitIdleError> {
782 let (user_closures, result) = self.poll_and_return_closures(poll_type);
783 user_closures.fire();
784 result
785 }
786
787 pub(crate) fn poll_and_return_closures(
794 &self,
795 poll_type: wgt::PollType<crate::SubmissionIndex>,
796 ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
797 let snatch_guard = self.snatchable_lock.read();
798 let maintain_result = self.maintain(poll_type, snatch_guard);
799
800 self.lose_if_oom();
801
802 self.deferred_resource_destruction();
805
806 maintain_result
807 }
808
809 pub(crate) fn maintain<'this>(
827 &'this self,
828 poll_type: wgt::PollType<crate::SubmissionIndex>,
829 snatch_guard: SnatchGuard,
830 ) -> (UserClosures, Result<wgt::PollStatus, WaitIdleError>) {
831 profiling::scope!("Device::maintain");
832
833 let mut user_closures = UserClosures::default();
834
835 let wait_submission_index = match poll_type {
837 wgt::PollType::Wait {
838 submission_index: Some(submission_index),
839 ..
840 } => {
841 let last_successful_submission_index = self
842 .last_successful_submission_index
843 .load(Ordering::Acquire);
844
845 if submission_index > last_successful_submission_index {
846 let result = Err(WaitIdleError::WrongSubmissionIndex(
847 submission_index,
848 last_successful_submission_index,
849 ));
850
851 return (user_closures, result);
852 }
853
854 Some(submission_index)
855 }
856 wgt::PollType::Wait {
857 submission_index: None,
858 ..
859 } => Some(
860 self.last_successful_submission_index
861 .load(Ordering::Acquire),
862 ),
863 wgt::PollType::Poll => None,
864 };
865
866 if let Some(target_submission_index) = wait_submission_index {
868 log::trace!("Device::maintain: waiting for submission index {target_submission_index}");
869
870 let wait_timeout = match poll_type {
871 wgt::PollType::Wait { timeout, .. } => timeout,
872 wgt::PollType::Poll => unreachable!(
873 "`wait_submission_index` index for poll type `Poll` should be None"
874 ),
875 };
876
877 let wait_result = unsafe {
878 self.raw()
879 .wait(self.fence.as_ref(), target_submission_index, wait_timeout)
880 };
881
882 if let Err(e) = wait_result {
885 let hal_error: WaitIdleError = self.handle_hal_error(e).into();
886 return (user_closures, Err(hal_error));
887 }
888 }
889
890 let fence_value_result = unsafe { self.raw().get_fence_value(self.fence.as_ref()) };
893 let current_finished_submission = match fence_value_result {
894 Ok(fence_value) => fence_value,
895 Err(e) => {
896 let hal_error: WaitIdleError = self.handle_hal_error(e).into();
897 return (user_closures, Err(hal_error));
898 }
899 };
900
901 let command_indices = self.command_indices.read();
903 let device_valid = self.is_valid();
908 drop(command_indices);
909
910 let mut queue_empty = false;
916 if let Some(queue) = self.get_queue() {
917 let queue_result = queue.maintain(current_finished_submission, &snatch_guard);
918 (
919 user_closures.submissions,
920 user_closures.mappings,
921 user_closures.blas_compact_ready,
922 queue_empty,
923 ) = queue_result;
924 drop(snatch_guard);
938 } else {
939 drop(snatch_guard);
940 };
941
942 let result = if queue_empty {
945 if let Some(wait_submission_index) = wait_submission_index {
946 assert!(
949 current_finished_submission >= wait_submission_index,
950 concat!(
951 "If the queue is empty, the current submission index ",
952 "({}) should be at least the wait submission index ({})",
953 ),
954 current_finished_submission,
955 wait_submission_index,
956 );
957 }
958
959 Ok(wgt::PollStatus::QueueEmpty)
960 } else if let Some(wait_submission_index) = wait_submission_index {
961 if current_finished_submission >= wait_submission_index {
965 Ok(wgt::PollStatus::WaitSucceeded)
966 } else {
967 Err(WaitIdleError::Timeout)
968 }
969 } else {
970 Ok(wgt::PollStatus::Poll)
971 };
972
973 let mut should_release_gpu_resource = false;
980 if !device_valid && queue_empty {
981 should_release_gpu_resource = true;
984
985 if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
988 user_closures
989 .device_lost_invocations
990 .push(DeviceLostInvocation {
991 closure: device_lost_closure,
992 reason: DeviceLostReason::Destroyed,
993 message: String::new(),
994 });
995 }
996 }
997
998 if should_release_gpu_resource {
999 self.release_gpu_resources();
1000 }
1001
1002 (user_closures, result)
1003 }
1004
1005 pub fn create_buffer(
1006 self: &Arc<Self>,
1007 desc: &resource::BufferDescriptor,
1008 ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
1009 self.check_is_valid()?;
1010
1011 if desc.size > self.limits.max_buffer_size {
1012 return Err(resource::CreateBufferError::MaxBufferSize {
1013 requested: desc.size,
1014 maximum: self.limits.max_buffer_size,
1015 });
1016 }
1017
1018 if desc
1019 .usage
1020 .intersects(wgt::BufferUsages::BLAS_INPUT | wgt::BufferUsages::TLAS_INPUT)
1021 {
1022 self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)?;
1023 }
1024
1025 if desc.usage.contains(wgt::BufferUsages::INDEX)
1026 && desc.usage.contains(
1027 wgt::BufferUsages::VERTEX
1028 | wgt::BufferUsages::UNIFORM
1029 | wgt::BufferUsages::INDIRECT
1030 | wgt::BufferUsages::STORAGE,
1031 )
1032 {
1033 self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
1034 }
1035
1036 if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1037 return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
1038 }
1039
1040 if !self
1041 .features
1042 .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
1043 {
1044 use wgt::BufferUsages as Bu;
1045 let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
1046 && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
1047 let read_mismatch = desc.usage.contains(Bu::MAP_READ)
1048 && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
1049 if write_mismatch || read_mismatch {
1050 return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
1051 }
1052 }
1053
1054 let mut usage = conv::map_buffer_usage(desc.usage);
1055
1056 if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
1057 self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
1058 usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
1061 }
1062
1063 if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
1064 usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
1065 }
1066
1067 if desc.mapped_at_creation {
1068 if !desc.size.is_multiple_of(wgt::COPY_BUFFER_ALIGNMENT) {
1069 return Err(resource::CreateBufferError::UnalignedSize);
1070 }
1071 if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1072 usage |= wgt::BufferUses::COPY_DST;
1074 }
1075 } else {
1076 usage |= wgt::BufferUses::COPY_DST;
1079 }
1080
1081 let actual_size = if desc.size == 0 {
1082 wgt::COPY_BUFFER_ALIGNMENT
1083 } else if desc.usage.contains(wgt::BufferUsages::VERTEX) {
1084 desc.size + 1
1087 } else {
1088 desc.size
1089 };
1090 let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT;
1091 let aligned_size = if clear_remainder != 0 {
1092 actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder
1093 } else {
1094 actual_size
1095 };
1096
1097 let hal_desc = hal::BufferDescriptor {
1098 label: desc.label.to_hal(self.instance_flags),
1099 size: aligned_size,
1100 usage,
1101 memory_flags: hal::MemoryFlags::empty(),
1102 };
1103 let buffer = unsafe { self.raw().create_buffer(&hal_desc) }
1104 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1105
1106 let timestamp_normalization_bind_group = Snatchable::new(unsafe {
1107 self.timestamp_normalizer
1109 .get()
1110 .unwrap()
1111 .create_normalization_bind_group(
1112 self,
1113 &*buffer,
1114 desc.label.as_deref(),
1115 wgt::BufferSize::new(hal_desc.size).unwrap(),
1116 desc.usage,
1117 )
1118 }?);
1119
1120 let indirect_validation_bind_groups =
1121 self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
1122
1123 let buffer = Buffer {
1124 raw: Snatchable::new(buffer),
1125 device: self.clone(),
1126 usage: desc.usage,
1127 size: desc.size,
1128 initialization_status: RwLock::new(
1129 rank::BUFFER_INITIALIZATION_STATUS,
1130 BufferInitTracker::new(aligned_size),
1131 ),
1132 map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1133 destroyed_while_mapped: OnceCellOrLock::new(),
1134 label: desc.label.to_string(),
1135 tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1136 bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1137 timestamp_normalization_bind_group,
1138 indirect_validation_bind_groups,
1139 };
1140
1141 let buffer = Arc::new(buffer);
1142
1143 let buffer_use = if !desc.mapped_at_creation {
1144 wgt::BufferUses::empty()
1145 } else if desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
1146 let map_size = buffer.size;
1148 let mapping = if map_size == 0 {
1149 hal::BufferMapping {
1150 ptr: core::ptr::NonNull::dangling(),
1151 is_coherent: true,
1152 }
1153 } else {
1154 let snatch_guard: SnatchGuard = self.snatchable_lock.read();
1155 map_buffer(&buffer, 0, map_size, HostMap::Write, &snatch_guard)?
1156 };
1157 *buffer.map_state.lock() = resource::BufferMapState::Active {
1158 mapping,
1159 range: 0..map_size,
1160 host: HostMap::Write,
1161 };
1162 wgt::BufferUses::MAP_WRITE
1163 } else {
1164 let mut staging_buffer =
1165 StagingBuffer::new(self, wgt::BufferSize::new(aligned_size).unwrap())?;
1166
1167 staging_buffer.write_zeros();
1170 buffer.initialization_status.write().drain(0..aligned_size);
1171
1172 *buffer.map_state.lock() = resource::BufferMapState::Init { staging_buffer };
1173 wgt::BufferUses::COPY_DST
1174 };
1175
1176 self.trackers
1177 .lock()
1178 .buffers
1179 .insert_single(&buffer, buffer_use);
1180
1181 Ok(buffer)
1182 }
1183
1184 #[cfg(feature = "replay")]
1185 pub fn set_buffer_data(
1186 self: &Arc<Self>,
1187 buffer: &Arc<Buffer>,
1188 offset: wgt::BufferAddress,
1189 data: &[u8],
1190 ) -> resource::BufferAccessResult {
1191 use crate::resource::RawResourceAccess;
1192
1193 let device = &buffer.device;
1194
1195 device.check_is_valid()?;
1196 buffer.check_usage(wgt::BufferUsages::MAP_WRITE)?;
1197
1198 let last_submission = device
1199 .get_queue()
1200 .and_then(|queue| queue.lock_life().get_buffer_latest_submission_index(buffer));
1201
1202 if let Some(last_submission) = last_submission {
1203 device.wait_for_submit(last_submission)?;
1204 }
1205
1206 let snatch_guard = device.snatchable_lock.read();
1207 let raw_buf = buffer.try_raw(&snatch_guard)?;
1208
1209 if offset > buffer.size {
1210 return Err(resource::BufferAccessError::OutOfBoundsStartOffsetOverrun {
1211 index: offset,
1212 max: buffer.size,
1213 });
1214 } else if buffer.size - offset < u64::try_from(data.len()).unwrap() {
1215 return Err(resource::BufferAccessError::OutOfBoundsEndOffsetOverrun {
1216 index: offset,
1217 size: u64::try_from(data.len()).unwrap(),
1218 max: buffer.size,
1219 });
1220 }
1221
1222 let mapping = unsafe {
1223 device
1224 .raw()
1225 .map_buffer(raw_buf, offset..offset + u64::try_from(data.len()).unwrap())
1226 }
1227 .map_err(|e| device.handle_hal_error(e))?;
1228
1229 unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), mapping.ptr.as_ptr(), data.len()) };
1230
1231 if !mapping.is_coherent {
1232 #[allow(clippy::single_range_in_vec_init)]
1233 unsafe {
1234 device
1235 .raw()
1236 .flush_mapped_ranges(raw_buf, &[offset..offset + data.len() as u64])
1237 };
1238 }
1239
1240 unsafe { device.raw().unmap_buffer(raw_buf) };
1241
1242 Ok(())
1243 }
1244
1245 pub(crate) fn create_texture_from_hal(
1246 self: &Arc<Self>,
1247 hal_texture: Box<dyn hal::DynTexture>,
1248 desc: &resource::TextureDescriptor,
1249 initial_state: wgt::TextureUses,
1250 ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1251 let format_features = self
1252 .describe_format_features(desc.format)
1253 .map_err(|error| resource::CreateTextureError::MissingFeatures(desc.format, error))?;
1254
1255 unsafe { self.raw().add_raw_texture(&*hal_texture) };
1256
1257 let texture = Texture::new(
1258 self,
1259 resource::TextureInner::Native { raw: hal_texture },
1260 conv::map_texture_usage(desc.usage, desc.format.into(), format_features.flags),
1261 desc,
1262 format_features,
1263 resource::TextureClearMode::None,
1264 false,
1265 );
1266
1267 let texture = Arc::new(texture);
1268
1269 self.trackers
1270 .lock()
1271 .textures
1272 .insert_single(&texture, initial_state);
1273
1274 Ok(texture)
1275 }
1276
1277 pub(crate) unsafe fn create_buffer_from_hal(
1284 self: &Arc<Self>,
1285 hal_buffer: Box<dyn hal::DynBuffer>,
1286 desc: &resource::BufferDescriptor,
1287 ) -> (Fallible<Buffer>, Option<resource::CreateBufferError>) {
1288 let timestamp_normalization_bind_group = unsafe {
1289 match self
1290 .timestamp_normalizer
1291 .get()
1292 .unwrap()
1293 .create_normalization_bind_group(
1294 self,
1295 &*hal_buffer,
1296 desc.label.as_deref(),
1297 wgt::BufferSize::new(desc.size).unwrap(),
1298 desc.usage,
1299 ) {
1300 Ok(bg) => Snatchable::new(bg),
1301 Err(e) => {
1302 return (
1303 Fallible::Invalid(Arc::new(desc.label.to_string())),
1304 Some(e.into()),
1305 )
1306 }
1307 }
1308 };
1309
1310 let indirect_validation_bind_groups = match self.create_indirect_validation_bind_groups(
1311 hal_buffer.as_ref(),
1312 desc.size,
1313 desc.usage,
1314 ) {
1315 Ok(ok) => ok,
1316 Err(e) => return (Fallible::Invalid(Arc::new(desc.label.to_string())), Some(e)),
1317 };
1318
1319 unsafe { self.raw().add_raw_buffer(&*hal_buffer) };
1320
1321 let buffer = Buffer {
1322 raw: Snatchable::new(hal_buffer),
1323 device: self.clone(),
1324 usage: desc.usage,
1325 size: desc.size,
1326 initialization_status: RwLock::new(
1327 rank::BUFFER_INITIALIZATION_STATUS,
1328 BufferInitTracker::new(0),
1329 ),
1330 map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
1331 destroyed_while_mapped: OnceCellOrLock::new(),
1332 label: desc.label.to_string(),
1333 tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
1334 bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
1335 timestamp_normalization_bind_group,
1336 indirect_validation_bind_groups,
1337 };
1338
1339 let buffer = Arc::new(buffer);
1340
1341 self.trackers
1342 .lock()
1343 .buffers
1344 .insert_single(&buffer, wgt::BufferUses::empty());
1345
1346 (Fallible::Valid(buffer), None)
1347 }
1348
1349 fn create_indirect_validation_bind_groups(
1350 &self,
1351 raw_buffer: &dyn hal::DynBuffer,
1352 buffer_size: u64,
1353 usage: wgt::BufferUsages,
1354 ) -> Result<Snatchable<crate::indirect_validation::BindGroups>, resource::CreateBufferError>
1355 {
1356 if !usage.contains(wgt::BufferUsages::INDIRECT) {
1357 return Ok(Snatchable::empty());
1358 }
1359
1360 let Some(ref indirect_validation) = self.indirect_validation else {
1361 return Ok(Snatchable::empty());
1362 };
1363
1364 let bind_groups = crate::indirect_validation::BindGroups::new(
1365 indirect_validation,
1366 self,
1367 buffer_size,
1368 raw_buffer,
1369 )
1370 .map_err(resource::CreateBufferError::IndirectValidationBindGroup)?;
1371
1372 if let Some(bind_groups) = bind_groups {
1373 Ok(Snatchable::new(bind_groups))
1374 } else {
1375 Ok(Snatchable::empty())
1376 }
1377 }
1378
1379 fn create_texture_inner(
1380 self: &Arc<Self>,
1381 desc: &resource::TextureDescriptor,
1382 ) -> Result<Arc<Texture>, resource::CreateTextureError> {
1383 use resource::{CreateTextureError, TextureDimensionError};
1384
1385 self.check_is_valid()?;
1386
1387 if desc.usage.is_empty() || desc.usage.contains_unknown_bits() {
1388 return Err(CreateTextureError::InvalidUsage(desc.usage));
1389 }
1390
1391 conv::check_texture_dimension_size(
1392 desc.dimension,
1393 desc.size,
1394 desc.sample_count,
1395 &self.limits,
1396 )?;
1397
1398 if desc.dimension != wgt::TextureDimension::D2 {
1399 if desc.format.is_depth_stencil_format() {
1401 return Err(CreateTextureError::InvalidDepthDimension(
1402 desc.dimension,
1403 desc.format,
1404 ));
1405 }
1406 if desc
1408 .usage
1409 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1410 {
1411 return Err(CreateTextureError::InvalidDimensionUsages(
1412 wgt::TextureUsages::TRANSIENT_ATTACHMENT,
1413 desc.dimension,
1414 ));
1415 }
1416 }
1417
1418 if desc.dimension != wgt::TextureDimension::D2
1419 && desc.dimension != wgt::TextureDimension::D3
1420 {
1421 if desc.format.is_compressed() {
1423 return Err(CreateTextureError::InvalidCompressedDimension(
1424 desc.dimension,
1425 desc.format,
1426 ));
1427 }
1428
1429 if desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1431 return Err(CreateTextureError::InvalidDimensionUsages(
1432 wgt::TextureUsages::RENDER_ATTACHMENT,
1433 desc.dimension,
1434 ));
1435 }
1436 }
1437
1438 if desc.format.is_compressed() {
1439 let (block_width, block_height) = desc.format.block_dimensions();
1440
1441 if !desc.size.width.is_multiple_of(block_width) {
1442 return Err(CreateTextureError::InvalidDimension(
1443 TextureDimensionError::NotMultipleOfBlockWidth {
1444 width: desc.size.width,
1445 block_width,
1446 format: desc.format,
1447 },
1448 ));
1449 }
1450
1451 if !desc.size.height.is_multiple_of(block_height) {
1452 return Err(CreateTextureError::InvalidDimension(
1453 TextureDimensionError::NotMultipleOfBlockHeight {
1454 height: desc.size.height,
1455 block_height,
1456 format: desc.format,
1457 },
1458 ));
1459 }
1460
1461 if desc.dimension == wgt::TextureDimension::D3 {
1462 if desc.format.is_bcn() {
1464 self.require_features(wgt::Features::TEXTURE_COMPRESSION_BC_SLICED_3D)
1465 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1466 } else if desc.format.is_astc() {
1467 self.require_features(wgt::Features::TEXTURE_COMPRESSION_ASTC_SLICED_3D)
1468 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1469 } else {
1470 return Err(CreateTextureError::InvalidCompressedDimension(
1471 desc.dimension,
1472 desc.format,
1473 ));
1474 }
1475 }
1476 }
1477
1478 let mips = desc.mip_level_count;
1479 let max_levels_allowed = desc.size.max_mips(desc.dimension).min(hal::MAX_MIP_LEVELS);
1480 if mips == 0 || mips > max_levels_allowed {
1481 return Err(CreateTextureError::InvalidMipLevelCount {
1482 requested: mips,
1483 maximum: max_levels_allowed,
1484 });
1485 }
1486
1487 {
1488 let (mut width_multiple, mut height_multiple) = desc.format.size_multiple_requirement();
1489
1490 if desc.format.is_multi_planar_format() {
1491 width_multiple <<= desc.mip_level_count.saturating_sub(1);
1495 height_multiple <<= desc.mip_level_count.saturating_sub(1);
1496 }
1497
1498 if !desc.size.width.is_multiple_of(width_multiple) {
1499 return Err(CreateTextureError::InvalidDimension(
1500 TextureDimensionError::WidthNotMultipleOf {
1501 width: desc.size.width,
1502 multiple: width_multiple,
1503 format: desc.format,
1504 },
1505 ));
1506 }
1507
1508 if !desc.size.height.is_multiple_of(height_multiple) {
1509 return Err(CreateTextureError::InvalidDimension(
1510 TextureDimensionError::HeightNotMultipleOf {
1511 height: desc.size.height,
1512 multiple: height_multiple,
1513 format: desc.format,
1514 },
1515 ));
1516 }
1517 }
1518
1519 if desc
1520 .usage
1521 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1522 {
1523 if desc.usage
1524 != (wgt::TextureUsages::TRANSIENT_ATTACHMENT
1525 | wgt::TextureUsages::RENDER_ATTACHMENT)
1526 {
1527 return Err(CreateTextureError::InvalidTransientTextureUsage(desc.usage));
1528 }
1529
1530 if desc.mip_level_count != 1 {
1531 return Err(CreateTextureError::InvalidTransientTextureMipLevelCount(
1532 desc.mip_level_count,
1533 ));
1534 }
1535
1536 if desc.size.depth_or_array_layers != 1 {
1537 return Err(CreateTextureError::InvalidTransientTextureLayerCount(
1538 desc.size.depth_or_array_layers,
1539 ));
1540 }
1541
1542 if !desc.view_formats.is_empty() {
1543 return Err(CreateTextureError::InvalidTransientTextureViewFormats);
1544 }
1545 }
1546
1547 let format_features = self
1548 .describe_format_features(desc.format)
1549 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1550
1551 if desc.sample_count > 1 {
1552 if desc.mip_level_count != 1 {
1558 return Err(CreateTextureError::InvalidMipLevelCount {
1559 requested: desc.mip_level_count,
1560 maximum: 1,
1561 });
1562 }
1563
1564 if desc.size.depth_or_array_layers != 1
1565 && !self.features.contains(wgt::Features::MULTISAMPLE_ARRAY)
1566 {
1567 return Err(CreateTextureError::InvalidDimension(
1568 TextureDimensionError::MultisampledDepthOrArrayLayer(
1569 desc.size.depth_or_array_layers,
1570 ),
1571 ));
1572 }
1573
1574 if desc.usage.contains(wgt::TextureUsages::STORAGE_BINDING) {
1575 return Err(CreateTextureError::InvalidMultisampledStorageBinding);
1576 }
1577
1578 if !desc.usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
1579 return Err(CreateTextureError::MultisampledNotRenderAttachment);
1580 }
1581
1582 if !format_features.flags.intersects(
1583 wgt::TextureFormatFeatureFlags::MULTISAMPLE_X4
1584 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X2
1585 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X8
1586 | wgt::TextureFormatFeatureFlags::MULTISAMPLE_X16,
1587 ) {
1588 return Err(CreateTextureError::InvalidMultisampledFormat(desc.format));
1589 }
1590
1591 if !format_features
1592 .flags
1593 .sample_count_supported(desc.sample_count)
1594 {
1595 return Err(CreateTextureError::InvalidSampleCount(
1596 desc.sample_count,
1597 desc.format,
1598 desc.format
1599 .guaranteed_format_features(self.features)
1600 .flags
1601 .supported_sample_counts(),
1602 self.adapter
1603 .get_texture_format_features(desc.format)
1604 .flags
1605 .supported_sample_counts(),
1606 ));
1607 };
1608 }
1609
1610 let missing_allowed_usages = match desc.format.planes() {
1611 Some(planes) => {
1612 let mut planes_usages = wgt::TextureUsages::all();
1613 for plane in 0..planes {
1614 let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1615 let format = desc.format.aspect_specific_format(aspect).unwrap();
1616 let format_features = self
1617 .describe_format_features(format)
1618 .map_err(|error| CreateTextureError::MissingFeatures(desc.format, error))?;
1619
1620 planes_usages &= format_features.allowed_usages;
1621 }
1622
1623 desc.usage - planes_usages
1624 }
1625 None => desc.usage - format_features.allowed_usages,
1626 };
1627
1628 if !missing_allowed_usages.is_empty() {
1629 let wgpu_allowed_usages = desc
1631 .format
1632 .guaranteed_format_features(self.features)
1633 .allowed_usages;
1634 let wgpu_missing_usages = desc.usage - wgpu_allowed_usages;
1635 return Err(CreateTextureError::InvalidFormatUsages(
1636 missing_allowed_usages,
1637 desc.format,
1638 wgpu_missing_usages.is_empty(),
1639 ));
1640 }
1641
1642 let mut hal_view_formats = Vec::new();
1643 for format in desc.view_formats.iter() {
1644 if desc.format == *format {
1645 continue;
1646 }
1647 if desc.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
1648 return Err(CreateTextureError::InvalidViewFormat(*format, desc.format));
1649 }
1650 hal_view_formats.push(*format);
1651 }
1652 if !hal_view_formats.is_empty() {
1653 self.require_downlevel_flags(wgt::DownlevelFlags::VIEW_FORMATS)?;
1654 }
1655
1656 let hal_usage = conv::map_texture_usage_for_texture(desc, &format_features);
1657
1658 let hal_desc = hal::TextureDescriptor {
1659 label: desc.label.to_hal(self.instance_flags),
1660 size: desc.size,
1661 mip_level_count: desc.mip_level_count,
1662 sample_count: desc.sample_count,
1663 dimension: desc.dimension,
1664 format: desc.format,
1665 usage: hal_usage,
1666 memory_flags: hal::MemoryFlags::empty(),
1667 view_formats: hal_view_formats,
1668 };
1669
1670 let raw_texture = unsafe { self.raw().create_texture(&hal_desc) }
1671 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
1672
1673 let clear_mode = if hal_usage
1674 .intersects(wgt::TextureUses::DEPTH_STENCIL_WRITE | wgt::TextureUses::COLOR_TARGET)
1675 && desc.dimension == wgt::TextureDimension::D2
1676 {
1677 let (is_color, usage) = if desc.format.is_depth_stencil_format() {
1678 (false, wgt::TextureUses::DEPTH_STENCIL_WRITE)
1679 } else {
1680 (true, wgt::TextureUses::COLOR_TARGET)
1681 };
1682
1683 let clear_label = hal_label(
1684 Some("(wgpu internal) clear texture view"),
1685 self.instance_flags,
1686 );
1687
1688 let mut clear_views = SmallVec::new();
1689 for mip_level in 0..desc.mip_level_count {
1690 for array_layer in 0..desc.size.depth_or_array_layers {
1691 macro_rules! push_clear_view {
1692 ($format:expr, $aspect:expr) => {
1693 let desc = hal::TextureViewDescriptor {
1694 label: clear_label,
1695 format: $format,
1696 dimension: TextureViewDimension::D2,
1697 usage,
1698 range: wgt::ImageSubresourceRange {
1699 aspect: $aspect,
1700 base_mip_level: mip_level,
1701 mip_level_count: Some(1),
1702 base_array_layer: array_layer,
1703 array_layer_count: Some(1),
1704 },
1705 };
1706 clear_views.push(ManuallyDrop::new(
1707 unsafe {
1708 self.raw().create_texture_view(raw_texture.as_ref(), &desc)
1709 }
1710 .map_err(|e| self.handle_hal_error(e))?,
1711 ));
1712 };
1713 }
1714
1715 if let Some(planes) = desc.format.planes() {
1716 for plane in 0..planes {
1717 let aspect = wgt::TextureAspect::from_plane(plane).unwrap();
1718 let format = desc.format.aspect_specific_format(aspect).unwrap();
1719 push_clear_view!(format, aspect);
1720 }
1721 } else {
1722 push_clear_view!(desc.format, wgt::TextureAspect::All);
1723 }
1724 }
1725 }
1726 resource::TextureClearMode::RenderPass {
1727 clear_views,
1728 is_color,
1729 }
1730 } else {
1731 resource::TextureClearMode::BufferCopy
1732 };
1733
1734 let texture = Texture::new(
1735 self,
1736 resource::TextureInner::Native { raw: raw_texture },
1737 hal_usage,
1738 desc,
1739 format_features,
1740 clear_mode,
1741 true,
1742 );
1743
1744 let texture = Arc::new(texture);
1745
1746 self.trackers
1747 .lock()
1748 .textures
1749 .insert_single(&texture, wgt::TextureUses::UNINITIALIZED);
1750
1751 Ok(texture)
1752 }
1753
1754 pub fn create_texture(
1755 self: &Arc<Self>,
1756 desc: &resource::TextureDescriptor,
1757 ) -> (Arc<Texture>, Option<resource::CreateTextureError>) {
1758 let (texture, error) = match self.create_texture_inner(desc) {
1759 Ok(texture) => (texture, None),
1760 Err(e) => {
1761 let texture = Texture::invalid(self, desc);
1762 (Arc::new(texture), Some(e))
1763 }
1764 };
1765 api_log!(
1766 "Device::create_texture({desc:?}) -> {:?}",
1767 Arc::as_ptr(&texture)
1768 );
1769
1770 #[cfg(feature = "trace")]
1771 if let Some(ref mut trace) = *self.trace.lock() {
1772 use crate::device::trace::IntoTrace as _;
1773
1774 trace.add(trace::Action::CreateTexture(
1775 texture.to_trace(),
1776 desc.clone(),
1777 ));
1778 }
1779 (texture, error)
1780 }
1781
1782 pub fn create_texture_error(
1784 self: &Arc<Self>,
1785 desc: &resource::TextureDescriptor,
1786 ) -> Arc<Texture> {
1787 let texture = Arc::new(Texture::invalid(self, desc));
1788 #[cfg(feature = "trace")]
1789 if let Some(ref mut trace) = *self.trace.lock() {
1790 use crate::device::trace::IntoTrace as _;
1791
1792 trace.add(trace::Action::CreateTextureError(
1793 texture.to_trace(),
1794 desc.clone(),
1795 ));
1796 }
1797 texture
1798 }
1799
1800 pub fn create_texture_view(
1801 self: &Arc<Self>,
1802 texture: &Arc<Texture>,
1803 desc: &resource::TextureViewDescriptor,
1804 ) -> Result<Arc<TextureView>, resource::CreateTextureViewError> {
1805 self.check_is_valid()?;
1806
1807 let snatch_guard = texture.device.snatchable_lock.read();
1808
1809 let texture_raw = texture.try_inner(&snatch_guard)?.raw();
1810
1811 let resolved_format = desc.format.unwrap_or_else(|| {
1814 texture
1815 .desc
1816 .format
1817 .aspect_specific_format(desc.range.aspect)
1818 .unwrap_or(texture.desc.format)
1819 });
1820
1821 let resolved_dimension = desc
1822 .dimension
1823 .unwrap_or_else(|| match texture.desc.dimension {
1824 wgt::TextureDimension::D1 => TextureViewDimension::D1,
1825 wgt::TextureDimension::D2 => {
1826 if texture.desc.array_layer_count() == 1 {
1827 TextureViewDimension::D2
1828 } else {
1829 TextureViewDimension::D2Array
1830 }
1831 }
1832 wgt::TextureDimension::D3 => TextureViewDimension::D3,
1833 });
1834
1835 let resolved_mip_level_count = desc.range.mip_level_count.unwrap_or_else(|| {
1836 texture
1837 .desc
1838 .mip_level_count
1839 .saturating_sub(desc.range.base_mip_level)
1840 });
1841
1842 let resolved_array_layer_count =
1843 desc.range
1844 .array_layer_count
1845 .unwrap_or_else(|| match resolved_dimension {
1846 TextureViewDimension::D1
1847 | TextureViewDimension::D2
1848 | TextureViewDimension::D3 => 1,
1849 TextureViewDimension::Cube => 6,
1850 TextureViewDimension::D2Array | TextureViewDimension::CubeArray => texture
1851 .desc
1852 .array_layer_count()
1853 .saturating_sub(desc.range.base_array_layer),
1854 });
1855
1856 let resolved_usage = {
1857 let usage = desc.usage.unwrap_or(wgt::TextureUsages::empty());
1858 if usage.is_empty() {
1859 texture.desc.usage
1860 } else if texture.desc.usage.contains(usage) {
1861 if texture
1863 .desc
1864 .usage
1865 .contains(wgt::TextureUsages::TRANSIENT_ATTACHMENT)
1866 && texture.desc.usage != usage
1867 {
1868 return Err(
1869 resource::CreateTextureViewError::InvalidTransientTextureViewUsage {
1870 texture: texture.desc.usage,
1871 view: usage,
1872 },
1873 );
1874 }
1875
1876 usage
1877 } else {
1878 return Err(resource::CreateTextureViewError::InvalidTextureViewUsage {
1879 view: usage,
1880 texture: texture.desc.usage,
1881 });
1882 }
1883 };
1884
1885 let format_features = self.describe_format_features(resolved_format)?;
1886 let allowed_format_usages = format_features.allowed_usages;
1887 if resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1888 && !allowed_format_usages.contains(wgt::TextureUsages::RENDER_ATTACHMENT)
1889 {
1890 return Err(
1891 resource::CreateTextureViewError::TextureViewFormatNotRenderable(resolved_format),
1892 );
1893 }
1894
1895 if resolved_usage.contains(wgt::TextureUsages::STORAGE_BINDING)
1896 && !allowed_format_usages.contains(wgt::TextureUsages::STORAGE_BINDING)
1897 {
1898 return Err(
1899 resource::CreateTextureViewError::TextureViewFormatNotStorage(resolved_format),
1900 );
1901 }
1902
1903 let aspects = hal::FormatAspects::new(texture.desc.format, desc.range.aspect);
1906 if aspects.is_empty() {
1907 return Err(resource::CreateTextureViewError::InvalidAspect {
1908 texture_format: texture.desc.format,
1909 requested_aspect: desc.range.aspect,
1910 });
1911 }
1912
1913 let format_is_good = if desc.range.aspect == wgt::TextureAspect::All {
1914 resolved_format == texture.desc.format
1915 || texture.desc.view_formats.contains(&resolved_format)
1916 } else {
1917 Some(resolved_format)
1918 == texture
1919 .desc
1920 .format
1921 .aspect_specific_format(desc.range.aspect)
1922 };
1923 if !format_is_good {
1924 return Err(resource::CreateTextureViewError::FormatReinterpretation {
1925 texture: texture.desc.format,
1926 view: resolved_format,
1927 });
1928 }
1929
1930 if texture.desc.sample_count > 1 && resolved_dimension != TextureViewDimension::D2 {
1932 let multisample_array_exception = resolved_dimension == TextureViewDimension::D2Array
1934 && self.features.contains(wgt::Features::MULTISAMPLE_ARRAY);
1935
1936 if !multisample_array_exception {
1937 return Err(
1938 resource::CreateTextureViewError::InvalidMultisampledTextureViewDimension(
1939 resolved_dimension,
1940 ),
1941 );
1942 }
1943 }
1944
1945 if texture.desc.dimension != resolved_dimension.compatible_texture_dimension() {
1947 return Err(
1948 resource::CreateTextureViewError::InvalidTextureViewDimension {
1949 view: resolved_dimension,
1950 texture: texture.desc.dimension,
1951 },
1952 );
1953 }
1954
1955 match resolved_dimension {
1956 TextureViewDimension::D1 | TextureViewDimension::D2 | TextureViewDimension::D3 => {
1957 if resolved_array_layer_count != 1 {
1958 return Err(resource::CreateTextureViewError::InvalidArrayLayerCount {
1959 requested: resolved_array_layer_count,
1960 dim: resolved_dimension,
1961 });
1962 }
1963 }
1964 TextureViewDimension::Cube => {
1965 if resolved_array_layer_count != 6 {
1966 return Err(
1967 resource::CreateTextureViewError::InvalidCubemapTextureDepth {
1968 depth: resolved_array_layer_count,
1969 },
1970 );
1971 }
1972 }
1973 TextureViewDimension::CubeArray => {
1974 if !resolved_array_layer_count.is_multiple_of(6) {
1975 return Err(
1976 resource::CreateTextureViewError::InvalidCubemapArrayTextureDepth {
1977 depth: resolved_array_layer_count,
1978 },
1979 );
1980 }
1981 }
1982 _ => {}
1983 }
1984
1985 match resolved_dimension {
1986 TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
1987 if texture.desc.size.width != texture.desc.size.height {
1988 return Err(resource::CreateTextureViewError::InvalidCubeTextureViewSize);
1989 }
1990 }
1991 _ => {}
1992 }
1993
1994 if resolved_mip_level_count == 0 {
1995 return Err(resource::CreateTextureViewError::ZeroMipLevelCount);
1996 }
1997
1998 let mip_level_end = desc
1999 .range
2000 .base_mip_level
2001 .saturating_add(resolved_mip_level_count);
2002
2003 let level_end = texture.desc.mip_level_count;
2004 if mip_level_end > level_end {
2005 return Err(resource::CreateTextureViewError::TooManyMipLevels {
2006 base_mip_level: desc.range.base_mip_level,
2007 mip_level_count: resolved_mip_level_count,
2008 total: level_end,
2009 });
2010 }
2011
2012 if resolved_array_layer_count == 0 {
2013 return Err(resource::CreateTextureViewError::ZeroArrayLayerCount);
2014 }
2015
2016 let array_layer_end = desc
2017 .range
2018 .base_array_layer
2019 .saturating_add(resolved_array_layer_count);
2020
2021 let layer_end = texture.desc.array_layer_count();
2022 if array_layer_end > layer_end {
2023 return Err(resource::CreateTextureViewError::TooManyArrayLayers {
2024 base_array_layer: desc.range.base_array_layer,
2025 array_layer_count: resolved_array_layer_count,
2026 total: layer_end,
2027 });
2028 };
2029
2030 let render_extent = 'error: {
2032 if !resolved_usage.contains(wgt::TextureUsages::RENDER_ATTACHMENT) {
2033 break 'error Err(TextureViewNotRenderableReason::Usage(resolved_usage));
2034 }
2035
2036 let allowed_view_dimensions = [
2037 TextureViewDimension::D2,
2038 TextureViewDimension::D2Array,
2039 TextureViewDimension::D3,
2040 ];
2041 if !allowed_view_dimensions.contains(&resolved_dimension) {
2042 break 'error Err(TextureViewNotRenderableReason::Dimension(
2043 resolved_dimension,
2044 ));
2045 }
2046
2047 if resolved_mip_level_count != 1 {
2048 break 'error Err(TextureViewNotRenderableReason::MipLevelCount(
2049 resolved_mip_level_count,
2050 ));
2051 }
2052
2053 if resolved_array_layer_count != 1
2054 && !(self.features.contains(wgt::Features::MULTIVIEW))
2055 {
2056 break 'error Err(TextureViewNotRenderableReason::ArrayLayerCount(
2057 resolved_array_layer_count,
2058 ));
2059 }
2060
2061 if !texture.desc.format.is_multi_planar_format()
2062 && aspects != hal::FormatAspects::from(texture.desc.format)
2063 {
2064 break 'error Err(TextureViewNotRenderableReason::Aspects(aspects));
2065 }
2066
2067 Ok(texture
2068 .desc
2069 .compute_render_extent(desc.range.base_mip_level, desc.range.aspect.to_plane()))
2070 };
2071
2072 let usage = {
2074 let resolved_hal_usage = conv::map_texture_usage(
2075 resolved_usage,
2076 resolved_format.into(),
2077 format_features.flags,
2078 );
2079 let mask_copy = !(wgt::TextureUses::COPY_SRC | wgt::TextureUses::COPY_DST);
2080 let mask_dimension = match resolved_dimension {
2081 TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2082 wgt::TextureUses::RESOURCE
2083 }
2084 TextureViewDimension::D3 => {
2085 wgt::TextureUses::RESOURCE
2086 | wgt::TextureUses::STORAGE_READ_ONLY
2087 | wgt::TextureUses::STORAGE_WRITE_ONLY
2088 | wgt::TextureUses::STORAGE_READ_WRITE
2089 }
2090 _ => wgt::TextureUses::all(),
2091 };
2092 let mask_mip_level = if resolved_mip_level_count == 1 {
2093 wgt::TextureUses::all()
2094 } else {
2095 wgt::TextureUses::RESOURCE
2096 };
2097 resolved_hal_usage & mask_copy & mask_dimension & mask_mip_level
2098 };
2099
2100 let format = if resolved_format.is_depth_stencil_component(texture.desc.format) {
2102 texture.desc.format
2103 } else {
2104 resolved_format
2105 };
2106
2107 let resolved_range = wgt::ImageSubresourceRange {
2108 aspect: desc.range.aspect,
2109 base_mip_level: desc.range.base_mip_level,
2110 mip_level_count: Some(resolved_mip_level_count),
2111 base_array_layer: desc.range.base_array_layer,
2112 array_layer_count: Some(resolved_array_layer_count),
2113 };
2114
2115 let hal_desc = hal::TextureViewDescriptor {
2116 label: desc.label.to_hal(self.instance_flags),
2117 format,
2118 dimension: resolved_dimension,
2119 usage,
2120 range: resolved_range,
2121 };
2122
2123 let raw = unsafe { self.raw().create_texture_view(texture_raw, &hal_desc) }
2124 .map_err(|e| self.handle_hal_error(e))?;
2125
2126 let selector = TextureSelector {
2127 mips: desc.range.base_mip_level..mip_level_end,
2128 layers: desc.range.base_array_layer..array_layer_end,
2129 };
2130
2131 let view = TextureView {
2132 raw: Snatchable::new(raw),
2133 parent: texture.clone(),
2134 device: self.clone(),
2135 desc: resource::HalTextureViewDescriptor {
2136 texture_format: texture.desc.format,
2137 format: resolved_format,
2138 dimension: resolved_dimension,
2139 usage: resolved_usage,
2140 range: resolved_range,
2141 },
2142 format_features: texture.format_features,
2143 render_extent,
2144 samples: texture.desc.sample_count,
2145 selector,
2146 label: desc.label.to_string(),
2147 };
2148
2149 let view = Arc::new(view);
2150
2151 {
2152 let mut views = texture.views.lock();
2153 views.push(Arc::downgrade(&view));
2154 }
2155
2156 Ok(view)
2157 }
2158
2159 pub fn create_external_texture(
2160 self: &Arc<Self>,
2161 desc: &resource::ExternalTextureDescriptor,
2162 planes: &[Arc<TextureView>],
2163 ) -> Result<Arc<ExternalTexture>, resource::CreateExternalTextureError> {
2164 use resource::CreateExternalTextureError;
2165 self.require_features(wgt::Features::EXTERNAL_TEXTURE)?;
2166 self.check_is_valid()?;
2167
2168 if desc.num_planes() != planes.len() {
2169 return Err(CreateExternalTextureError::IncorrectPlaneCount {
2170 format: desc.format,
2171 expected: desc.num_planes(),
2172 provided: planes.len(),
2173 });
2174 }
2175
2176 let planes = planes
2177 .iter()
2178 .enumerate()
2179 .map(|(i, plane)| {
2180 if plane.samples != 1 {
2181 return Err(CreateExternalTextureError::InvalidPlaneMultisample(
2182 plane.samples,
2183 ));
2184 }
2185
2186 let sample_type = plane
2187 .desc
2188 .format
2189 .sample_type(Some(plane.desc.range.aspect), Some(self.features))
2190 .unwrap();
2191 if !matches!(sample_type, TextureSampleType::Float { filterable: true }) {
2192 return Err(CreateExternalTextureError::InvalidPlaneSampleType {
2193 format: plane.desc.format,
2194 sample_type,
2195 });
2196 }
2197
2198 if plane.desc.dimension != TextureViewDimension::D2 {
2199 return Err(CreateExternalTextureError::InvalidPlaneDimension(
2200 plane.desc.dimension,
2201 ));
2202 }
2203
2204 let expected_components = match desc.format {
2205 wgt::ExternalTextureFormat::Rgba => 4,
2206 wgt::ExternalTextureFormat::Nv12 => match i {
2207 0 => 1,
2208 1 => 2,
2209 _ => unreachable!(),
2210 },
2211 wgt::ExternalTextureFormat::Yu12 => 1,
2212 };
2213 if plane.desc.format.components() != expected_components {
2214 return Err(CreateExternalTextureError::InvalidPlaneFormat {
2215 format: desc.format,
2216 plane: i,
2217 expected: expected_components,
2218 provided: plane.desc.format,
2219 });
2220 }
2221
2222 plane.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
2223 Ok(plane.clone())
2224 })
2225 .collect::<Result<_, _>>()?;
2226
2227 let params_data = ExternalTextureParams::from_desc(desc);
2228 let label = desc.label.as_ref().map(|l| alloc::format!("{l} params"));
2229 let params_desc = resource::BufferDescriptor {
2230 label: label.map(Cow::Owned),
2231 size: size_of_val(¶ms_data) as wgt::BufferAddress,
2232 usage: wgt::BufferUsages::UNIFORM | wgt::BufferUsages::COPY_DST,
2233 mapped_at_creation: false,
2234 };
2235 let params = self.create_buffer(¶ms_desc)?;
2236 self.get_queue().unwrap().write_buffer(
2237 params.clone(),
2238 0,
2239 bytemuck::bytes_of(¶ms_data),
2240 )?;
2241
2242 let external_texture = ExternalTexture {
2243 device: self.clone(),
2244 planes,
2245 params,
2246 label: desc.label.to_string(),
2247 tracking_data: TrackingData::new(self.tracker_indices.external_textures.clone()),
2248 };
2249 let external_texture = Arc::new(external_texture);
2250
2251 Ok(external_texture)
2252 }
2253
2254 pub fn create_sampler(
2255 self: &Arc<Self>,
2256 desc: &resource::SamplerDescriptor,
2257 ) -> Result<Arc<Sampler>, resource::CreateSamplerError> {
2258 self.check_is_valid()?;
2259
2260 if desc
2261 .address_modes
2262 .iter()
2263 .any(|am| am == &wgt::AddressMode::ClampToBorder)
2264 {
2265 self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_BORDER)?;
2266 }
2267
2268 if desc.border_color == Some(wgt::SamplerBorderColor::Zero) {
2269 self.require_features(wgt::Features::ADDRESS_MODE_CLAMP_TO_ZERO)?;
2270 }
2271
2272 if desc.lod_min_clamp < 0.0 {
2273 return Err(resource::CreateSamplerError::InvalidLodMinClamp(
2274 desc.lod_min_clamp,
2275 ));
2276 }
2277 if desc.lod_max_clamp < desc.lod_min_clamp {
2278 return Err(resource::CreateSamplerError::InvalidLodMaxClamp {
2279 lod_min_clamp: desc.lod_min_clamp,
2280 lod_max_clamp: desc.lod_max_clamp,
2281 });
2282 }
2283
2284 if desc.anisotropy_clamp < 1 {
2285 return Err(resource::CreateSamplerError::InvalidAnisotropy(
2286 desc.anisotropy_clamp,
2287 ));
2288 }
2289
2290 if desc.anisotropy_clamp != 1 {
2291 if !matches!(desc.min_filter, wgt::FilterMode::Linear) {
2292 return Err(
2293 resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2294 filter_type: resource::SamplerFilterErrorType::MinFilter,
2295 filter_mode: desc.min_filter,
2296 anisotropic_clamp: desc.anisotropy_clamp,
2297 },
2298 );
2299 }
2300 if !matches!(desc.mag_filter, wgt::FilterMode::Linear) {
2301 return Err(
2302 resource::CreateSamplerError::InvalidFilterModeWithAnisotropy {
2303 filter_type: resource::SamplerFilterErrorType::MagFilter,
2304 filter_mode: desc.mag_filter,
2305 anisotropic_clamp: desc.anisotropy_clamp,
2306 },
2307 );
2308 }
2309 if !matches!(desc.mipmap_filter, wgt::MipmapFilterMode::Linear) {
2310 return Err(
2311 resource::CreateSamplerError::InvalidMipmapFilterModeWithAnisotropy {
2312 filter_type: resource::SamplerFilterErrorType::MipmapFilter,
2313 filter_mode: desc.mipmap_filter,
2314 anisotropic_clamp: desc.anisotropy_clamp,
2315 },
2316 );
2317 }
2318 }
2319
2320 let anisotropy_clamp = if self
2321 .downlevel
2322 .flags
2323 .contains(wgt::DownlevelFlags::ANISOTROPIC_FILTERING)
2324 {
2325 desc.anisotropy_clamp.min(16)
2327 } else {
2328 1
2330 };
2331
2332 let hal_desc = hal::SamplerDescriptor {
2335 label: desc.label.to_hal(self.instance_flags),
2336 address_modes: desc.address_modes,
2337 mag_filter: desc.mag_filter,
2338 min_filter: desc.min_filter,
2339 mipmap_filter: desc.mipmap_filter,
2340 lod_clamp: desc.lod_min_clamp..desc.lod_max_clamp,
2341 compare: desc.compare,
2342 anisotropy_clamp,
2343 border_color: desc.border_color,
2344 };
2345
2346 let raw = unsafe { self.raw().create_sampler(&hal_desc) }
2347 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
2348
2349 let sampler = Sampler {
2350 raw: ManuallyDrop::new(raw),
2351 device: self.clone(),
2352 label: desc.label.to_string(),
2353 tracking_data: TrackingData::new(self.tracker_indices.samplers.clone()),
2354 comparison: desc.compare.is_some(),
2355 filtering: desc.min_filter == wgt::FilterMode::Linear
2356 || desc.mag_filter == wgt::FilterMode::Linear
2357 || desc.mipmap_filter == wgt::MipmapFilterMode::Linear,
2358 };
2359
2360 let sampler = Arc::new(sampler);
2361
2362 Ok(sampler)
2363 }
2364
2365 pub fn create_shader_module<'a>(
2366 self: &Arc<Self>,
2367 desc: &pipeline::ShaderModuleDescriptor<'a>,
2368 source: pipeline::ShaderModuleSource<'a>,
2369 ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2370 self.check_is_valid()?;
2371
2372 let (module, source) = match source {
2373 #[cfg(feature = "wgsl")]
2374 pipeline::ShaderModuleSource::Wgsl(code) => {
2375 profiling::scope!("naga::front::wgsl::parse");
2376 let capabilities =
2377 features_to_naga_capabilities(self.features, self.downlevel.flags);
2378 let mut options = naga::front::wgsl::Options::new();
2379 options.capabilities = capabilities;
2380 let mut frontend = naga::front::wgsl::Frontend::new_with_options(options);
2381 let module = frontend.parse(&code).map_err(|inner| {
2382 pipeline::CreateShaderModuleError::Parsing(naga::error::ShaderError {
2383 source: code.to_string(),
2384 label: desc.label.as_ref().map(|l| l.to_string()),
2385 inner: Box::new(inner),
2386 })
2387 })?;
2388 (Cow::Owned(module), code.into_owned())
2389 }
2390 #[cfg(feature = "spirv")]
2391 pipeline::ShaderModuleSource::SpirV(spv, options) => {
2392 let parser = naga::front::spv::Frontend::new(spv.iter().cloned(), &options);
2393 profiling::scope!("naga::front::spv::Frontend");
2394 let module = parser.parse().map_err(|inner| {
2395 pipeline::CreateShaderModuleError::ParsingSpirV(naga::error::ShaderError {
2396 source: String::new(),
2397 label: desc.label.as_ref().map(|l| l.to_string()),
2398 inner: Box::new(inner),
2399 })
2400 })?;
2401 (Cow::Owned(module), String::new())
2402 }
2403 #[cfg(feature = "glsl")]
2404 pipeline::ShaderModuleSource::Glsl(code, options) => {
2405 let mut parser = naga::front::glsl::Frontend::default();
2406 profiling::scope!("naga::front::glsl::Frontend.parse");
2407 let module = parser.parse(&options, &code).map_err(|inner| {
2408 pipeline::CreateShaderModuleError::ParsingGlsl(naga::error::ShaderError {
2409 source: code.to_string(),
2410 label: desc.label.as_ref().map(|l| l.to_string()),
2411 inner: Box::new(inner),
2412 })
2413 })?;
2414 (Cow::Owned(module), code.into_owned())
2415 }
2416 pipeline::ShaderModuleSource::Naga(module) => (module, String::new()),
2417 pipeline::ShaderModuleSource::Dummy(_) => panic!("found `ShaderModuleSource::Dummy`"),
2418 };
2419 for (_, var) in module.global_variables.iter() {
2420 match var.binding {
2421 Some(br) if br.group >= self.limits.max_bind_groups => {
2422 return Err(pipeline::CreateShaderModuleError::InvalidGroupIndex {
2423 bind: br,
2424 group: br.group,
2425 limit: self.limits.max_bind_groups,
2426 });
2427 }
2428 _ => continue,
2429 };
2430 }
2431
2432 profiling::scope!("naga::validate");
2433 let debug_source =
2434 if self.instance_flags.contains(wgt::InstanceFlags::DEBUG) && !source.is_empty() {
2435 Some(hal::DebugSource {
2436 file_name: Cow::Owned(
2437 desc.label
2438 .as_ref()
2439 .map_or("shader".to_string(), |l| l.to_string()),
2440 ),
2441 source_code: Cow::Owned(source.clone()),
2442 })
2443 } else {
2444 None
2445 };
2446
2447 let info = create_validator(
2448 self.features,
2449 self.downlevel.flags,
2450 naga::valid::ValidationFlags::all(),
2451 )
2452 .validate(&module)
2453 .map_err(|inner| {
2454 pipeline::CreateShaderModuleError::Validation(naga::error::ShaderError {
2455 source,
2456 label: desc.label.as_ref().map(|l| l.to_string()),
2457 inner: Box::new(inner),
2458 })
2459 })?;
2460
2461 let interface = validation::Interface::new(&module, &info, self.limits.clone());
2462 let hal_shader = hal::ShaderInput::Naga(hal::NagaShader {
2463 module,
2464 info,
2465 debug_source,
2466 });
2467 let hal_desc = hal::ShaderModuleDescriptor {
2468 label: desc.label.to_hal(self.instance_flags),
2469 runtime_checks: desc.runtime_checks,
2470 };
2471 let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2472 Ok(raw) => raw,
2473 Err(error) => {
2474 return Err(match error {
2475 hal::ShaderError::Device(error) => {
2476 pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2477 }
2478 hal::ShaderError::Compilation(ref msg) => {
2479 log::error!("Shader error: {msg}");
2480 pipeline::CreateShaderModuleError::Generation
2481 }
2482 })
2483 }
2484 };
2485
2486 let module = pipeline::ShaderModule {
2487 raw: ManuallyDrop::new(raw),
2488 device: self.clone(),
2489 interface: ShaderMetaData::Interface(interface),
2490 label: desc.label.to_string(),
2491 };
2492
2493 let module = Arc::new(module);
2494
2495 Ok(module)
2496 }
2497
2498 #[allow(unused_unsafe)]
2500 #[doc(hidden)]
2501 pub unsafe fn create_shader_module_passthrough<'a>(
2502 self: &Arc<Self>,
2503 descriptor: &pipeline::ShaderModuleDescriptorPassthrough<'a>,
2504 ) -> Result<Arc<pipeline::ShaderModule>, pipeline::CreateShaderModuleError> {
2505 self.check_is_valid()?;
2506 self.require_features(wgt::Features::PASSTHROUGH_SHADERS)?;
2507
2508 if (descriptor.dxil.is_some() || descriptor.glsl.is_some())
2510 && descriptor.entry_points.len() != 1
2511 {
2512 return Err(pipeline::CreateShaderModuleError::IncorrectPassthroughEntryPointCount);
2513 }
2514
2515 let entry_point_hashmap = || {
2516 descriptor
2517 .entry_points
2518 .iter()
2519 .map(|e| (e.name.to_string(), e.workgroup_size))
2520 .collect()
2521 };
2522
2523 let hal_shader = match self.backend() {
2524 wgt::Backend::Vulkan => hal::ShaderInput::SpirV(
2525 descriptor
2526 .spirv
2527 .as_ref()
2528 .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2529 ),
2530 wgt::Backend::Dx12 => {
2531 if let Some(dxil) = &descriptor.dxil {
2532 hal::ShaderInput::Dxil { shader: dxil }
2533 } else if let Some(hlsl) = &descriptor.hlsl {
2534 hal::ShaderInput::Hlsl { shader: hlsl }
2535 } else {
2536 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2537 }
2538 }
2539 wgt::Backend::Metal => {
2540 if let Some(metallib) = &descriptor.metallib {
2541 hal::ShaderInput::MetalLib {
2542 file: metallib,
2543 num_workgroups: entry_point_hashmap(),
2544 }
2545 } else if let Some(msl) = &descriptor.msl {
2546 hal::ShaderInput::Msl {
2547 shader: msl,
2548 num_workgroups: entry_point_hashmap(),
2549 }
2550 } else {
2551 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend);
2552 }
2553 }
2554 wgt::Backend::Gl => hal::ShaderInput::Glsl {
2555 shader: descriptor
2556 .glsl
2557 .as_ref()
2558 .ok_or(pipeline::CreateShaderModuleError::NotCompiledForBackend)?,
2559 },
2560 wgt::Backend::Noop => {
2561 return Err(pipeline::CreateShaderModuleError::NotCompiledForBackend)
2562 }
2563 wgt::Backend::BrowserWebGpu => unreachable!(),
2564 };
2565
2566 let hal_desc = hal::ShaderModuleDescriptor {
2567 label: descriptor.label.to_hal(self.instance_flags),
2568 runtime_checks: wgt::ShaderRuntimeChecks::unchecked(),
2569 };
2570
2571 let raw = match unsafe { self.raw().create_shader_module(&hal_desc, hal_shader) } {
2572 Ok(raw) => raw,
2573 Err(error) => {
2574 return Err(match error {
2575 hal::ShaderError::Device(error) => {
2576 pipeline::CreateShaderModuleError::Device(self.handle_hal_error(error))
2577 }
2578 hal::ShaderError::Compilation(ref msg) => {
2579 log::error!("Shader error: {msg}");
2580 pipeline::CreateShaderModuleError::Generation
2581 }
2582 })
2583 }
2584 };
2585
2586 let module = pipeline::ShaderModule {
2587 raw: ManuallyDrop::new(raw),
2588 device: self.clone(),
2589 interface: ShaderMetaData::Passthrough(PassthroughInterface {
2590 entry_point_names: descriptor
2591 .entry_points
2592 .iter()
2593 .map(|e| e.name.to_string())
2594 .collect(),
2595 }),
2596 label: descriptor.label.to_string(),
2597 };
2598
2599 Ok(Arc::new(module))
2600 }
2601
2602 pub(crate) fn create_command_encoder(
2603 self: &Arc<Self>,
2604 label: &crate::Label,
2605 ) -> Result<Arc<command::CommandEncoder>, DeviceError> {
2606 self.check_is_valid()?;
2607
2608 let queue = self.get_queue().unwrap();
2609
2610 let encoder = self
2611 .command_allocator
2612 .acquire_encoder(self.raw(), queue.raw())
2613 .map_err(|e| self.handle_hal_error(e))?;
2614
2615 let cmd_enc = command::CommandEncoder::new(encoder, self, label);
2616
2617 let cmd_enc = Arc::new(cmd_enc);
2618
2619 Ok(cmd_enc)
2620 }
2621
2622 fn make_late_sized_buffer_groups(
2625 shader_binding_sizes: &FastHashMap<naga::ResourceBinding, wgt::BufferSize>,
2626 layout: &binding_model::PipelineLayout,
2627 ) -> ArrayVec<pipeline::LateSizedBufferGroup, { hal::MAX_BIND_GROUPS }> {
2628 layout
2632 .bind_group_layouts
2633 .iter()
2634 .enumerate()
2635 .map(|(group_index, bgl)| {
2636 let Some(bgl) = bgl else {
2637 return pipeline::LateSizedBufferGroup::default();
2638 };
2639
2640 let shader_sizes = bgl
2641 .entries
2642 .values()
2643 .filter_map(|entry| match entry.ty {
2644 wgt::BindingType::Buffer {
2645 min_binding_size: None,
2646 ..
2647 } => {
2648 let rb = naga::ResourceBinding {
2649 group: group_index as u32,
2650 binding: entry.binding,
2651 };
2652 let shader_size =
2653 shader_binding_sizes.get(&rb).map_or(0, |nz| nz.get());
2654 Some(shader_size)
2655 }
2656 _ => None,
2657 })
2658 .collect();
2659 pipeline::LateSizedBufferGroup { shader_sizes }
2660 })
2661 .collect()
2662 }
2663
2664 pub fn create_bind_group_layout(
2665 self: &Arc<Self>,
2666 desc: &binding_model::BindGroupLayoutDescriptor,
2667 ) -> (Arc<BindGroupLayout>, Option<CreateBindGroupLayoutError>) {
2668 let (bgl, error) = match self.create_bind_group_layout_inner(desc) {
2669 Ok(layout) => (layout, None),
2670 Err(e) => (
2671 BindGroupLayout::invalid(self, desc.label.to_string()),
2672 Some(e),
2673 ),
2674 };
2675 #[cfg(feature = "trace")]
2676 if let Some(ref mut trace) = *self.trace.lock() {
2677 use crate::device::trace::IntoTrace;
2678
2679 trace.add(trace::Action::CreateBindGroupLayout(
2680 bgl.to_trace(),
2681 desc.clone(),
2682 ));
2683 }
2684 (bgl, error)
2685 }
2686
2687 fn create_bind_group_layout_inner(
2688 self: &Arc<Device>,
2689 desc: &binding_model::BindGroupLayoutDescriptor,
2690 ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2691 self.check_is_valid()?;
2692
2693 let entry_map = bgl::EntryMap::from_entries(&desc.entries)?;
2694
2695 let bgl_result = self.bgl_pool.get_or_init(entry_map, |entry_map| {
2696 let bgl =
2697 self.create_bind_group_layout_impl(&desc.label, entry_map, bgl::Origin::Pool)?;
2698 bgl.exclusive_pipeline
2699 .set(binding_model::ExclusivePipeline::None)
2700 .unwrap();
2701 Ok(bgl)
2702 });
2703
2704 match bgl_result {
2705 Ok(layout) => Ok(layout),
2706 Err(e) => Err(e),
2707 }
2708 }
2709
2710 fn create_bind_group_layout_impl(
2711 self: &Arc<Self>,
2712 label: &crate::Label,
2713 entry_map: bgl::EntryMap,
2714 origin: bgl::Origin,
2715 ) -> Result<Arc<BindGroupLayout>, CreateBindGroupLayoutError> {
2716 #[derive(PartialEq)]
2717 enum WritableStorage {
2718 Yes,
2719 No,
2720 }
2721
2722 for entry in entry_map.values() {
2723 if entry.binding >= self.limits.max_bindings_per_bind_group {
2724 return Err(CreateBindGroupLayoutError::InvalidBindingIndex {
2725 binding: entry.binding,
2726 maximum: self.limits.max_bindings_per_bind_group,
2727 });
2728 }
2729
2730 use wgt::BindingType as Bt;
2731
2732 let mut required_features = wgt::Features::empty();
2733 let mut required_downlevel_flags = wgt::DownlevelFlags::empty();
2734 let (array_feature, writable_storage) = match entry.ty {
2735 Bt::Buffer {
2736 ty: wgt::BufferBindingType::Uniform,
2737 has_dynamic_offset: false,
2738 min_binding_size: _,
2739 } => (
2740 Some(wgt::Features::BUFFER_BINDING_ARRAY),
2741 WritableStorage::No,
2742 ),
2743 Bt::Buffer {
2744 ty: wgt::BufferBindingType::Uniform,
2745 has_dynamic_offset: true,
2746 min_binding_size: _,
2747 } => (
2748 Some(wgt::Features::BUFFER_BINDING_ARRAY),
2749 WritableStorage::No,
2750 ),
2751 Bt::Buffer {
2752 ty: wgt::BufferBindingType::Storage { read_only },
2753 ..
2754 } => (
2755 Some(
2756 wgt::Features::BUFFER_BINDING_ARRAY
2757 | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2758 ),
2759 match read_only {
2760 true => WritableStorage::No,
2761 false => WritableStorage::Yes,
2762 },
2763 ),
2764 Bt::Sampler { .. } => (
2765 Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2766 WritableStorage::No,
2767 ),
2768 Bt::Texture {
2769 multisampled: true,
2770 sample_type: TextureSampleType::Float { filterable: true },
2771 ..
2772 } => {
2773 return Err(CreateBindGroupLayoutError::Entry {
2774 binding: entry.binding,
2775 error:
2776 BindGroupLayoutEntryError::SampleTypeFloatFilterableBindingMultisampled,
2777 });
2778 }
2779 Bt::Texture {
2780 multisampled,
2781 view_dimension,
2782 ..
2783 } => {
2784 if multisampled && view_dimension != TextureViewDimension::D2 {
2785 return Err(CreateBindGroupLayoutError::Entry {
2786 binding: entry.binding,
2787 error: BindGroupLayoutEntryError::Non2DMultisampled(view_dimension),
2788 });
2789 }
2790
2791 (
2792 Some(wgt::Features::TEXTURE_BINDING_ARRAY),
2793 WritableStorage::No,
2794 )
2795 }
2796 Bt::StorageTexture {
2797 access,
2798 view_dimension,
2799 format,
2800 } => {
2801 use wgt::{StorageTextureAccess as Access, TextureFormatFeatureFlags as Flags};
2802
2803 match view_dimension {
2804 TextureViewDimension::Cube | TextureViewDimension::CubeArray => {
2805 return Err(CreateBindGroupLayoutError::Entry {
2806 binding: entry.binding,
2807 error: BindGroupLayoutEntryError::StorageTextureCube,
2808 })
2809 }
2810 _ => (),
2811 }
2812 match access {
2813 wgt::StorageTextureAccess::Atomic
2814 if !self.features.contains(wgt::Features::TEXTURE_ATOMIC) =>
2815 {
2816 return Err(CreateBindGroupLayoutError::Entry {
2817 binding: entry.binding,
2818 error: BindGroupLayoutEntryError::StorageTextureAtomic,
2819 });
2820 }
2821 _ => (),
2822 }
2823
2824 let format_features =
2825 self.describe_format_features(format).map_err(|error| {
2826 CreateBindGroupLayoutError::Entry {
2827 binding: entry.binding,
2828 error: BindGroupLayoutEntryError::MissingFeatures(error),
2829 }
2830 })?;
2831
2832 let required_feature_flag = match access {
2833 Access::WriteOnly => Flags::STORAGE_WRITE_ONLY,
2834 Access::ReadOnly => Flags::STORAGE_READ_ONLY,
2835 Access::ReadWrite => Flags::STORAGE_READ_WRITE,
2836 Access::Atomic => Flags::STORAGE_ATOMIC,
2837 };
2838
2839 if !format_features.flags.contains(required_feature_flag) {
2840 return Err(
2841 CreateBindGroupLayoutError::UnsupportedStorageTextureAccess {
2842 binding: entry.binding,
2843 access,
2844 format,
2845 },
2846 );
2847 }
2848
2849 (
2850 Some(
2851 wgt::Features::TEXTURE_BINDING_ARRAY
2852 | wgt::Features::STORAGE_RESOURCE_BINDING_ARRAY,
2853 ),
2854 match access {
2855 wgt::StorageTextureAccess::WriteOnly => WritableStorage::Yes,
2856 wgt::StorageTextureAccess::ReadOnly => WritableStorage::No,
2857 wgt::StorageTextureAccess::ReadWrite => WritableStorage::Yes,
2858 wgt::StorageTextureAccess::Atomic => {
2859 required_features |= wgt::Features::TEXTURE_ATOMIC;
2860 WritableStorage::Yes
2861 }
2862 },
2863 )
2864 }
2865 Bt::AccelerationStructure { vertex_return } => {
2866 self.require_features(wgt::Features::EXPERIMENTAL_RAY_QUERY)
2867 .map_err(|e| CreateBindGroupLayoutError::Entry {
2868 binding: entry.binding,
2869 error: e.into(),
2870 })?;
2871 if vertex_return {
2872 self.require_features(wgt::Features::EXPERIMENTAL_RAY_HIT_VERTEX_RETURN)
2873 .map_err(|e| CreateBindGroupLayoutError::Entry {
2874 binding: entry.binding,
2875 error: e.into(),
2876 })?;
2877 }
2878 (
2879 Some(wgt::Features::ACCELERATION_STRUCTURE_BINDING_ARRAY),
2880 WritableStorage::No,
2881 )
2882 }
2883 Bt::ExternalTexture => {
2884 self.require_features(wgt::Features::EXTERNAL_TEXTURE)
2885 .map_err(|e| CreateBindGroupLayoutError::Entry {
2886 binding: entry.binding,
2887 error: e.into(),
2888 })?;
2889 (None, WritableStorage::No)
2890 }
2891 };
2892
2893 if entry.count.is_some() {
2895 required_features |= array_feature
2896 .ok_or(BindGroupLayoutEntryError::ArrayUnsupported)
2897 .map_err(|error| CreateBindGroupLayoutError::Entry {
2898 binding: entry.binding,
2899 error,
2900 })?;
2901 }
2902
2903 if entry.visibility.contains_unknown_bits() {
2904 return Err(CreateBindGroupLayoutError::InvalidVisibility(
2905 entry.visibility,
2906 ));
2907 }
2908
2909 if entry.visibility.contains(wgt::ShaderStages::VERTEX) {
2910 if writable_storage == WritableStorage::Yes {
2911 required_features |= wgt::Features::VERTEX_WRITABLE_STORAGE;
2912 }
2913 if let Bt::Buffer {
2914 ty: wgt::BufferBindingType::Storage { .. },
2915 ..
2916 } = entry.ty
2917 {
2918 required_downlevel_flags |= wgt::DownlevelFlags::VERTEX_STORAGE;
2919 }
2920 }
2921 if writable_storage == WritableStorage::Yes
2922 && entry.visibility.contains(wgt::ShaderStages::FRAGMENT)
2923 {
2924 required_downlevel_flags |= wgt::DownlevelFlags::FRAGMENT_WRITABLE_STORAGE;
2925 }
2926
2927 self.require_features(required_features)
2928 .map_err(BindGroupLayoutEntryError::MissingFeatures)
2929 .map_err(|error| CreateBindGroupLayoutError::Entry {
2930 binding: entry.binding,
2931 error,
2932 })?;
2933 self.require_downlevel_flags(required_downlevel_flags)
2934 .map_err(BindGroupLayoutEntryError::MissingDownlevelFlags)
2935 .map_err(|error| CreateBindGroupLayoutError::Entry {
2936 binding: entry.binding,
2937 error,
2938 })?;
2939 }
2940
2941 let bgl_flags = conv::bind_group_layout_flags(self.features);
2942
2943 let hal_bindings = entry_map.values().copied().collect::<Vec<_>>();
2944 let hal_desc = hal::BindGroupLayoutDescriptor {
2945 label: label.to_hal(self.instance_flags),
2946 flags: bgl_flags,
2947 entries: &hal_bindings,
2948 };
2949
2950 let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
2951 for entry in entry_map.values() {
2952 count_validator.add_binding(entry);
2953 }
2954 count_validator
2957 .validate(&self.limits, self.instance_flags)
2958 .map_err(CreateBindGroupLayoutError::TooManyBindings)?;
2959
2960 count_validator.validate_binding_arrays()?;
2962
2963 let raw = unsafe { self.raw().create_bind_group_layout(&hal_desc) }
2964 .map_err(|e| self.handle_hal_error(e))?;
2965
2966 let bgl = BindGroupLayout {
2967 state: ResourceState::Valid(BindGroupLayoutState {
2968 raw: binding_model::RawBindGroupLayout::Owning(ManuallyDrop::new(raw)),
2969 origin,
2970 binding_count_validator: count_validator,
2971 }),
2972 device: self.clone(),
2973 entries: entry_map,
2974 exclusive_pipeline: OnceCellOrLock::new(),
2975 label: label.to_string(),
2976 };
2977
2978 let bgl = Arc::new(bgl);
2979
2980 Ok(bgl)
2981 }
2982
2983 fn create_buffer_binding<'a>(
2984 &self,
2985 bb: &'a binding_model::ResolvedBufferBinding,
2986 binding: u32,
2987 decl: &wgt::BindGroupLayoutEntry,
2988 buffer_init_actions: &mut Vec<BufferInitTrackerAction>,
2989 dynamic_binding_info: &mut Vec<binding_model::BindGroupDynamicBindingData>,
2990 late_buffer_binding_sizes: &mut FastHashMap<u32, wgt::BufferSize>,
2991 used: &mut BindGroupStates,
2992 snatch_guard: &'a SnatchGuard<'a>,
2993 ) -> Result<hal::BufferBinding<'a, dyn hal::DynBuffer>, CreateBindGroupError> {
2994 use crate::binding_model::CreateBindGroupError as Error;
2995
2996 let (binding_ty, dynamic, min_size) = match decl.ty {
2997 wgt::BindingType::Buffer {
2998 ty,
2999 has_dynamic_offset,
3000 min_binding_size,
3001 } => (ty, has_dynamic_offset, min_binding_size),
3002 _ => {
3003 return Err(Error::WrongBindingType {
3004 binding,
3005 actual: decl.ty,
3006 expected: "UniformBuffer, StorageBuffer or ReadonlyStorageBuffer",
3007 })
3008 }
3009 };
3010
3011 let (pub_usage, internal_use, range_limit) = match binding_ty {
3012 wgt::BufferBindingType::Uniform => (
3013 wgt::BufferUsages::UNIFORM,
3014 wgt::BufferUses::UNIFORM,
3015 self.limits.max_uniform_buffer_binding_size,
3016 ),
3017 wgt::BufferBindingType::Storage { read_only } => (
3018 wgt::BufferUsages::STORAGE,
3019 if read_only {
3020 wgt::BufferUses::STORAGE_READ_ONLY
3021 } else {
3022 wgt::BufferUses::STORAGE_READ_WRITE
3023 },
3024 self.limits.max_storage_buffer_binding_size,
3025 ),
3026 };
3027
3028 let (align, align_limit_name) =
3029 binding_model::buffer_binding_type_alignment(&self.limits, binding_ty);
3030 if !bb.offset.is_multiple_of(align as u64) {
3031 return Err(Error::UnalignedBufferOffset(
3032 bb.offset,
3033 align_limit_name,
3034 align,
3035 ));
3036 }
3037
3038 let buffer = &bb.buffer;
3039
3040 used.buffers.insert_single(buffer.clone(), internal_use);
3041
3042 buffer.same_device(self)?;
3043
3044 buffer.check_usage(pub_usage)?;
3045
3046 let req_size = match bb.size.map(wgt::BufferSize::new) {
3047 Some(non_zero @ Some(_)) => non_zero,
3049 None => None,
3051 Some(None) => return Err(CreateBindGroupError::BindingZeroSize(buffer.error_ident())),
3053 };
3054 let (bb, bind_size) = buffer.binding(bb.offset, req_size, snatch_guard)?;
3055
3056 if matches!(binding_ty, wgt::BufferBindingType::Storage { .. })
3057 && bind_size % u64::from(wgt::STORAGE_BINDING_SIZE_ALIGNMENT) != 0
3058 {
3059 return Err(Error::UnalignedEffectiveBufferBindingSizeForStorage {
3060 alignment: wgt::STORAGE_BINDING_SIZE_ALIGNMENT,
3061 size: bind_size,
3062 });
3063 }
3064
3065 let bind_end = bb.offset + bind_size;
3066
3067 if bind_size > range_limit {
3068 return Err(Error::BufferRangeTooLarge {
3069 binding,
3070 given: bind_size,
3071 limit: range_limit,
3072 });
3073 }
3074
3075 if dynamic {
3077 dynamic_binding_info.push(binding_model::BindGroupDynamicBindingData {
3078 binding_idx: binding,
3079 buffer_size: buffer.size,
3080 binding_range: bb.offset..bind_end,
3081 maximum_dynamic_offset: buffer.size - bind_end,
3082 binding_type: binding_ty,
3083 });
3084 }
3085
3086 if let Some(non_zero) = min_size {
3087 let min_size = non_zero.get();
3088 if min_size > bind_size {
3089 return Err(Error::BindingSizeTooSmall {
3090 buffer: buffer.error_ident(),
3091 actual: bind_size,
3092 min: min_size,
3093 });
3094 }
3095 } else {
3096 let late_size = wgt::BufferSize::new(bind_size)
3097 .ok_or_else(|| Error::BindingZeroSize(buffer.error_ident()))?;
3098 late_buffer_binding_sizes.insert(binding, late_size);
3099 }
3100
3101 assert_eq!(bb.offset % wgt::COPY_BUFFER_ALIGNMENT, 0);
3104
3105 let init_range = if dynamic {
3106 0..buffer.size
3109 } else {
3110 let bounds_check_alignment = binding_model::buffer_binding_type_bounds_check_alignment(
3115 &self.alignments,
3116 binding_ty,
3117 );
3118 let visible_size = align_to(bind_size, bounds_check_alignment);
3119
3120 bb.offset..bb.offset + visible_size
3121 };
3122
3123 buffer_init_actions.extend(buffer.initialization_status.read().create_action(
3124 buffer,
3125 init_range,
3126 MemoryInitKind::NeedsInitializedMemory,
3127 ));
3128
3129 Ok(bb)
3130 }
3131
3132 fn create_sampler_binding<'a>(
3133 &self,
3134 used: &mut BindGroupStates,
3135 binding: u32,
3136 decl: &wgt::BindGroupLayoutEntry,
3137 sampler: &'a Arc<Sampler>,
3138 ) -> Result<&'a dyn hal::DynSampler, CreateBindGroupError> {
3139 use crate::binding_model::CreateBindGroupError as Error;
3140
3141 used.samplers.insert_single(sampler.clone());
3142
3143 sampler.same_device(self)?;
3144
3145 match decl.ty {
3146 wgt::BindingType::Sampler(ty) => {
3147 let (allowed_filtering, allowed_comparison) = match ty {
3148 wgt::SamplerBindingType::Filtering => (None, false),
3149 wgt::SamplerBindingType::NonFiltering => (Some(false), false),
3150 wgt::SamplerBindingType::Comparison => (None, true),
3151 };
3152 if let Some(allowed_filtering) = allowed_filtering {
3153 if allowed_filtering != sampler.filtering {
3154 return Err(Error::WrongSamplerFiltering {
3155 binding,
3156 layout_flt: allowed_filtering,
3157 sampler_flt: sampler.filtering,
3158 });
3159 }
3160 }
3161 if allowed_comparison != sampler.comparison {
3162 return Err(Error::WrongSamplerComparison {
3163 binding,
3164 layout_cmp: allowed_comparison,
3165 sampler_cmp: sampler.comparison,
3166 });
3167 }
3168 }
3169 _ => {
3170 return Err(Error::WrongBindingType {
3171 binding,
3172 actual: decl.ty,
3173 expected: "Sampler",
3174 })
3175 }
3176 }
3177
3178 Ok(sampler.raw())
3179 }
3180
3181 fn create_texture_binding<'a>(
3182 &self,
3183 binding: u32,
3184 decl: &wgt::BindGroupLayoutEntry,
3185 view: &'a Arc<TextureView>,
3186 used: &mut BindGroupStates,
3187 texture_init_actions: &mut Vec<TextureInitTrackerAction>,
3188 snatch_guard: &'a SnatchGuard<'a>,
3189 ) -> Result<hal::TextureBinding<'a, dyn hal::DynTextureView>, CreateBindGroupError> {
3190 view.same_device(self)?;
3191
3192 let internal_use = self.texture_use_parameters(
3193 binding,
3194 decl,
3195 view,
3196 "SampledTexture, ReadonlyStorageTexture or WriteonlyStorageTexture",
3197 )?;
3198
3199 used.views.insert_single(view.clone(), internal_use);
3200
3201 let texture = &view.parent;
3202
3203 texture_init_actions.push(TextureInitTrackerAction {
3204 texture: texture.clone(),
3205 range: TextureInitRange {
3206 mip_range: view.desc.range.mip_range(texture.desc.mip_level_count),
3207 layer_range: view
3208 .desc
3209 .range
3210 .layer_range(texture.desc.array_layer_count()),
3211 },
3212 kind: MemoryInitKind::NeedsInitializedMemory,
3213 });
3214
3215 Ok(hal::TextureBinding {
3216 view: view.try_raw(snatch_guard)?,
3217 usage: internal_use,
3218 })
3219 }
3220
3221 fn create_tlas_binding<'a>(
3222 self: &Arc<Self>,
3223 used: &mut BindGroupStates,
3224 binding: u32,
3225 decl: &wgt::BindGroupLayoutEntry,
3226 tlas: &'a Arc<Tlas>,
3227 snatch_guard: &'a SnatchGuard<'a>,
3228 ) -> Result<&'a dyn hal::DynAccelerationStructure, CreateBindGroupError> {
3229 use crate::binding_model::CreateBindGroupError as Error;
3230
3231 used.acceleration_structures.insert_single(tlas.clone());
3232
3233 tlas.same_device(self)?;
3234
3235 match decl.ty {
3236 wgt::BindingType::AccelerationStructure { vertex_return } => {
3237 if vertex_return
3238 && !tlas.flags.contains(
3239 wgpu_types::AccelerationStructureFlags::ALLOW_RAY_HIT_VERTEX_RETURN,
3240 )
3241 {
3242 return Err(Error::MissingTLASVertexReturn { binding });
3243 }
3244 }
3245 _ => {
3246 return Err(Error::WrongBindingType {
3247 binding,
3248 actual: decl.ty,
3249 expected: "Tlas",
3250 });
3251 }
3252 }
3253
3254 Ok(tlas.try_raw(snatch_guard)?)
3255 }
3256
3257 fn create_external_texture_binding<'a>(
3258 &'a self,
3259 binding: u32,
3260 decl: &wgt::BindGroupLayoutEntry,
3261 external_texture: &'a Arc<ExternalTexture>,
3262 used: &mut BindGroupStates,
3263 snatch_guard: &'a SnatchGuard,
3264 ) -> Result<
3265 hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3266 CreateBindGroupError,
3267 > {
3268 use crate::binding_model::CreateBindGroupError as Error;
3269
3270 external_texture.same_device(self)?;
3271
3272 used.external_textures
3273 .insert_single(external_texture.clone());
3274
3275 match decl.ty {
3276 wgt::BindingType::ExternalTexture => {}
3277 _ => {
3278 return Err(Error::WrongBindingType {
3279 binding,
3280 actual: decl.ty,
3281 expected: "ExternalTexture",
3282 });
3283 }
3284 }
3285
3286 let planes = (0..3)
3287 .map(|i| {
3288 let plane = external_texture
3292 .planes
3293 .get(i)
3294 .unwrap_or(&external_texture.planes[0]);
3295 let internal_use = wgt::TextureUses::RESOURCE;
3296 used.views.insert_single(plane.clone(), internal_use);
3297 let view = plane.try_raw(snatch_guard)?;
3298 Ok(hal::TextureBinding {
3299 view,
3300 usage: internal_use,
3301 })
3302 })
3303 .collect::<Result<Vec<_>, Error>>()?;
3306 let planes = planes.try_into().unwrap();
3307
3308 used.buffers
3309 .insert_single(external_texture.params.clone(), wgt::BufferUses::UNIFORM);
3310 let params = external_texture.params.binding(0, None, snatch_guard)?.0;
3311
3312 Ok(hal::ExternalTextureBinding { planes, params })
3313 }
3314
3315 fn create_external_texture_binding_from_view<'a>(
3316 &'a self,
3317 binding: u32,
3318 decl: &wgt::BindGroupLayoutEntry,
3319 view: &'a Arc<TextureView>,
3320 used: &mut BindGroupStates,
3321 snatch_guard: &'a SnatchGuard,
3322 ) -> Result<
3323 hal::ExternalTextureBinding<'a, dyn hal::DynBuffer, dyn hal::DynTextureView>,
3324 CreateBindGroupError,
3325 > {
3326 use crate::binding_model::CreateBindGroupError as Error;
3327
3328 view.same_device(self)?;
3329
3330 let internal_use = self.texture_use_parameters(binding, decl, view, "SampledTexture")?;
3331 used.views.insert_single(view.clone(), internal_use);
3332
3333 match decl.ty {
3334 wgt::BindingType::ExternalTexture => {}
3335 _ => {
3336 return Err(Error::WrongBindingType {
3337 binding,
3338 actual: decl.ty,
3339 expected: "ExternalTexture",
3340 });
3341 }
3342 }
3343
3344 let planes = [
3346 hal::TextureBinding {
3347 view: view.try_raw(snatch_guard)?,
3348 usage: internal_use,
3349 },
3350 hal::TextureBinding {
3351 view: view.try_raw(snatch_guard)?,
3352 usage: internal_use,
3353 },
3354 hal::TextureBinding {
3355 view: view.try_raw(snatch_guard)?,
3356 usage: internal_use,
3357 },
3358 ];
3359 let params = hal::BufferBinding::new_unchecked(
3360 self.default_external_texture_params_buffer.as_ref(),
3361 0,
3362 None,
3363 );
3364
3365 Ok(hal::ExternalTextureBinding { planes, params })
3366 }
3367
3368 pub fn create_bind_group(
3371 self: &Arc<Self>,
3372 desc: binding_model::ResolvedBindGroupDescriptor,
3373 ) -> Result<Arc<BindGroup>, CreateBindGroupError> {
3374 use crate::binding_model::{CreateBindGroupError as Error, ResolvedBindingResource as Br};
3375
3376 let layout = desc.layout;
3377
3378 self.check_is_valid()?;
3379 layout.same_device(self)?;
3380 layout.check_is_valid()?;
3381
3382 {
3383 let actual = desc.entries.len();
3386 let expected = layout.entries.len();
3387 if actual != expected {
3388 return Err(Error::BindingsNumMismatch { expected, actual });
3389 }
3390 }
3391
3392 let mut dynamic_binding_info = Vec::new();
3395 let mut late_buffer_binding_sizes = FastHashMap::default();
3399 let mut used = BindGroupStates::new();
3401
3402 let mut buffer_init_actions = Vec::new();
3403 let mut texture_init_actions = Vec::new();
3404 let mut hal_entries = Vec::with_capacity(desc.entries.len());
3405 let mut hal_buffers = Vec::new();
3406 let mut hal_samplers = Vec::new();
3407 let mut hal_textures = Vec::new();
3408 let mut hal_tlas_s = Vec::new();
3409 let mut hal_external_textures = Vec::new();
3410 let snatch_guard = self.snatchable_lock.read();
3411 for entry in desc.entries.iter() {
3412 let binding = entry.binding;
3413 let decl = layout
3415 .entries
3416 .get(binding)
3417 .ok_or(Error::MissingBindingDeclaration(binding))?;
3418 let (res_index, count) = match entry.resource {
3419 Br::Buffer(ref bb) => {
3420 let bb = self.create_buffer_binding(
3421 bb,
3422 binding,
3423 decl,
3424 &mut buffer_init_actions,
3425 &mut dynamic_binding_info,
3426 &mut late_buffer_binding_sizes,
3427 &mut used,
3428 &snatch_guard,
3429 )?;
3430
3431 let res_index = hal_buffers.len();
3432 hal_buffers.push(bb);
3433 (res_index, 1)
3434 }
3435 Br::BufferArray(ref bindings_array) => {
3436 let num_bindings = bindings_array.len();
3437 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3438
3439 let res_index = hal_buffers.len();
3440 for bb in bindings_array.iter() {
3441 let bb = self.create_buffer_binding(
3442 bb,
3443 binding,
3444 decl,
3445 &mut buffer_init_actions,
3446 &mut dynamic_binding_info,
3447 &mut late_buffer_binding_sizes,
3448 &mut used,
3449 &snatch_guard,
3450 )?;
3451 hal_buffers.push(bb);
3452 }
3453 (res_index, num_bindings)
3454 }
3455 Br::Sampler(ref sampler) => {
3456 let sampler = self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3457
3458 let res_index = hal_samplers.len();
3459 hal_samplers.push(sampler);
3460 (res_index, 1)
3461 }
3462 Br::SamplerArray(ref samplers) => {
3463 let num_bindings = samplers.len();
3464 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3465
3466 let res_index = hal_samplers.len();
3467 for sampler in samplers.iter() {
3468 let sampler =
3469 self.create_sampler_binding(&mut used, binding, decl, sampler)?;
3470
3471 hal_samplers.push(sampler);
3472 }
3473
3474 (res_index, num_bindings)
3475 }
3476 Br::TextureView(ref view) => match decl.ty {
3477 wgt::BindingType::ExternalTexture => {
3478 let et = self.create_external_texture_binding_from_view(
3479 binding,
3480 decl,
3481 view,
3482 &mut used,
3483 &snatch_guard,
3484 )?;
3485 let res_index = hal_external_textures.len();
3486 hal_external_textures.push(et);
3487 (res_index, 1)
3488 }
3489 _ => {
3490 let tb = self.create_texture_binding(
3491 binding,
3492 decl,
3493 view,
3494 &mut used,
3495 &mut texture_init_actions,
3496 &snatch_guard,
3497 )?;
3498 let res_index = hal_textures.len();
3499 hal_textures.push(tb);
3500 (res_index, 1)
3501 }
3502 },
3503 Br::TextureViewArray(ref views) => {
3504 let num_bindings = views.len();
3505 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3506
3507 let res_index = hal_textures.len();
3508 for view in views.iter() {
3509 let tb = self.create_texture_binding(
3510 binding,
3511 decl,
3512 view,
3513 &mut used,
3514 &mut texture_init_actions,
3515 &snatch_guard,
3516 )?;
3517
3518 hal_textures.push(tb);
3519 }
3520
3521 (res_index, num_bindings)
3522 }
3523 Br::AccelerationStructure(ref tlas) => {
3524 let tlas =
3525 self.create_tlas_binding(&mut used, binding, decl, tlas, &snatch_guard)?;
3526 let res_index = hal_tlas_s.len();
3527 hal_tlas_s.push(tlas);
3528 (res_index, 1)
3529 }
3530 Br::AccelerationStructureArray(ref tlas_array) => {
3531 let num_bindings = tlas_array.len();
3535 Self::check_array_binding(self.features, decl.count, num_bindings)?;
3536
3537 let res_index = hal_tlas_s.len();
3538 for tlas in tlas_array.iter() {
3539 let tlas = self.create_tlas_binding(
3540 &mut used,
3541 binding,
3542 decl,
3543 tlas,
3544 &snatch_guard,
3545 )?;
3546 hal_tlas_s.push(tlas);
3547 }
3548 (res_index, num_bindings)
3549 }
3550 Br::ExternalTexture(ref et) => {
3551 let et = self.create_external_texture_binding(
3552 binding,
3553 decl,
3554 et,
3555 &mut used,
3556 &snatch_guard,
3557 )?;
3558 let res_index = hal_external_textures.len();
3559 hal_external_textures.push(et);
3560 (res_index, 1)
3561 }
3562 };
3563
3564 hal_entries.push(hal::BindGroupEntry {
3565 binding,
3566 resource_index: res_index as u32,
3567 count: count as u32,
3568 });
3569 }
3570
3571 used.optimize();
3572
3573 hal_entries.sort_by_key(|entry| entry.binding);
3574 for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) {
3575 if a.binding == b.binding {
3576 return Err(Error::DuplicateBinding(a.binding));
3577 }
3578 }
3579
3580 dynamic_binding_info.sort_by_key(|i| i.binding_idx);
3581
3582 let hal_desc = hal::BindGroupDescriptor {
3583 label: desc.label.to_hal(self.instance_flags),
3584 layout: layout.try_raw()?,
3585 entries: &hal_entries,
3586 buffers: &hal_buffers,
3587 samplers: &hal_samplers,
3588 textures: &hal_textures,
3589 acceleration_structures: &hal_tlas_s,
3590 external_textures: &hal_external_textures,
3591 };
3592 let raw = unsafe { self.raw().create_bind_group(&hal_desc) }
3593 .map_err(|e| self.handle_hal_error(e))?;
3594
3595 let late_buffer_binding_infos = layout
3597 .entries
3598 .indices()
3599 .flat_map(|binding| {
3600 let size = late_buffer_binding_sizes.get(&binding).cloned()?;
3601 Some(BindGroupLateBufferBindingInfo {
3602 binding_index: binding,
3603 size,
3604 })
3605 })
3606 .collect();
3607
3608 let bind_group = BindGroup {
3609 raw: Snatchable::new(raw),
3610 device: self.clone(),
3611 layout,
3612 label: desc.label.to_string(),
3613 tracking_data: TrackingData::new(self.tracker_indices.bind_groups.clone()),
3614 used,
3615 buffer_init_actions,
3616 texture_init_actions,
3617 dynamic_binding_info,
3618 late_buffer_binding_infos,
3619 };
3620
3621 let bind_group = Arc::new(bind_group);
3622
3623 let weak_ref = Arc::downgrade(&bind_group);
3624 for texture in bind_group.used.views.used_textures() {
3625 let mut bind_groups = texture.bind_groups.lock();
3626 bind_groups.push(weak_ref.clone());
3627 }
3628 for buffer in bind_group.used.buffers.used_resources() {
3629 let mut bind_groups = buffer.bind_groups.lock();
3630 bind_groups.push(weak_ref.clone());
3631 }
3632
3633 Ok(bind_group)
3634 }
3635
3636 fn check_array_binding(
3637 features: wgt::Features,
3638 count: Option<NonZeroU32>,
3639 num_bindings: usize,
3640 ) -> Result<(), CreateBindGroupError> {
3641 use super::binding_model::CreateBindGroupError as Error;
3642
3643 if let Some(count) = count {
3644 let count = count.get() as usize;
3645 if count < num_bindings {
3646 return Err(Error::BindingArrayPartialLengthMismatch {
3647 actual: num_bindings,
3648 expected: count,
3649 });
3650 }
3651 if count != num_bindings
3652 && !features.contains(wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY)
3653 {
3654 return Err(Error::BindingArrayLengthMismatch {
3655 actual: num_bindings,
3656 expected: count,
3657 });
3658 }
3659 if num_bindings == 0 {
3660 return Err(Error::BindingArrayZeroLength);
3661 }
3662 } else {
3663 return Err(Error::SingleBindingExpected);
3664 };
3665
3666 Ok(())
3667 }
3668
3669 fn texture_use_parameters(
3670 &self,
3671 binding: u32,
3672 decl: &wgt::BindGroupLayoutEntry,
3673 view: &TextureView,
3674 expected: &'static str,
3675 ) -> Result<wgt::TextureUses, CreateBindGroupError> {
3676 use crate::binding_model::CreateBindGroupError as Error;
3677 if view
3678 .desc
3679 .aspects()
3680 .contains(hal::FormatAspects::DEPTH | hal::FormatAspects::STENCIL)
3681 {
3682 return Err(Error::DepthStencilAspect);
3683 }
3684 match decl.ty {
3685 wgt::BindingType::Texture {
3686 sample_type,
3687 view_dimension,
3688 multisampled,
3689 } => {
3690 use wgt::TextureSampleType as Tst;
3691 if multisampled != (view.samples != 1) {
3692 return Err(Error::InvalidTextureMultisample {
3693 binding,
3694 layout_multisampled: multisampled,
3695 view_samples: view.samples,
3696 });
3697 }
3698 let compat_sample_type = view
3699 .desc
3700 .format
3701 .sample_type(Some(view.desc.range.aspect), Some(self.features))
3702 .unwrap();
3703 match (sample_type, compat_sample_type) {
3704 (Tst::Uint, Tst::Uint) |
3705 (Tst::Sint, Tst::Sint) |
3706 (Tst::Depth, Tst::Depth) |
3707 (Tst::Float { filterable: false }, Tst::Float { .. }) |
3709 (Tst::Float { filterable: true }, Tst::Float { filterable: true }) |
3711 (Tst::Float { filterable: false }, Tst::Depth) => {}
3713 (Tst::Float { filterable: true }, Tst::Float { .. })
3718 if view.format_features.flags
3719 .contains(wgt::TextureFormatFeatureFlags::FILTERABLE) => {}
3720 _ => {
3721 return Err(Error::InvalidTextureSampleType {
3722 binding,
3723 layout_sample_type: sample_type,
3724 view_format: view.desc.format,
3725 view_sample_type: compat_sample_type,
3726 })
3727 }
3728 }
3729 if view_dimension != view.desc.dimension {
3730 return Err(Error::InvalidTextureDimension {
3731 binding,
3732 layout_dimension: view_dimension,
3733 view_dimension: view.desc.dimension,
3734 });
3735 }
3736 view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3737 Ok(wgt::TextureUses::RESOURCE)
3738 }
3739 wgt::BindingType::StorageTexture {
3740 access,
3741 format,
3742 view_dimension,
3743 } => {
3744 if format != view.desc.format {
3745 return Err(Error::InvalidStorageTextureFormat {
3746 binding,
3747 layout_format: format,
3748 view_format: view.desc.format,
3749 });
3750 }
3751 if view_dimension != view.desc.dimension {
3752 return Err(Error::InvalidTextureDimension {
3753 binding,
3754 layout_dimension: view_dimension,
3755 view_dimension: view.desc.dimension,
3756 });
3757 }
3758
3759 let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3760 if mip_level_count != 1 {
3761 return Err(Error::InvalidStorageTextureMipLevelCount {
3762 binding,
3763 mip_level_count,
3764 });
3765 }
3766
3767 view.check_usage(wgt::TextureUsages::STORAGE_BINDING)?;
3768
3769 Ok(match access {
3770 wgt::StorageTextureAccess::ReadOnly => wgt::TextureUses::STORAGE_READ_ONLY,
3771 wgt::StorageTextureAccess::WriteOnly => wgt::TextureUses::STORAGE_WRITE_ONLY,
3772 wgt::StorageTextureAccess::ReadWrite => wgt::TextureUses::STORAGE_READ_WRITE,
3773 wgt::StorageTextureAccess::Atomic => wgt::TextureUses::STORAGE_ATOMIC,
3774 })
3775 }
3776 wgt::BindingType::ExternalTexture => {
3777 if view.desc.dimension != TextureViewDimension::D2 {
3778 return Err(Error::InvalidTextureDimension {
3779 binding,
3780 layout_dimension: TextureViewDimension::D2,
3781 view_dimension: view.desc.dimension,
3782 });
3783 }
3784 let mip_level_count = view.selector.mips.end - view.selector.mips.start;
3785 if mip_level_count != 1 {
3786 return Err(Error::InvalidExternalTextureMipLevelCount {
3787 binding,
3788 mip_level_count,
3789 });
3790 }
3791 if view.desc.format != TextureFormat::Rgba8Unorm
3792 && view.desc.format != TextureFormat::Bgra8Unorm
3793 && view.desc.format != TextureFormat::Rgba16Float
3794 {
3795 return Err(Error::InvalidExternalTextureFormat {
3796 binding,
3797 format: view.desc.format,
3798 });
3799 }
3800 if view.samples != 1 {
3801 return Err(Error::InvalidTextureMultisample {
3802 binding,
3803 layout_multisampled: false,
3804 view_samples: view.samples,
3805 });
3806 }
3807
3808 view.check_usage(wgt::TextureUsages::TEXTURE_BINDING)?;
3809 Ok(wgt::TextureUses::RESOURCE)
3810 }
3811 _ => Err(Error::WrongBindingType {
3812 binding,
3813 actual: decl.ty,
3814 expected,
3815 }),
3816 }
3817 }
3818
3819 pub fn create_pipeline_layout(
3820 self: &Arc<Self>,
3821 desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3822 ) -> (
3823 Arc<binding_model::PipelineLayout>,
3824 Option<binding_model::CreatePipelineLayoutError>,
3825 ) {
3826 let (layout, error) = match self.create_pipeline_layout_impl(desc, false) {
3827 Ok(layout) => (layout, None),
3828 Err(e) => (
3829 binding_model::PipelineLayout::invalid(Arc::clone(self), desc.label.to_string()),
3830 Some(e),
3831 ),
3832 };
3833 #[cfg(feature = "trace")]
3834 if let Some(ref mut trace) = *self.trace.lock() {
3835 use crate::device::trace::IntoTrace;
3836 trace.add(trace::Action::CreatePipelineLayout(
3837 layout.to_trace(),
3838 desc.to_trace(),
3839 ));
3840 }
3841 api_log!(
3842 "Device::create_pipeline_layout -> {:?}",
3843 Arc::as_ptr(&layout)
3844 );
3845 (layout, error)
3846 }
3847
3848 fn create_pipeline_layout_impl(
3849 self: &Arc<Self>,
3850 desc: &binding_model::ResolvedPipelineLayoutDescriptor,
3851 ignore_exclusive_pipeline_check: bool,
3852 ) -> Result<Arc<binding_model::PipelineLayout>, binding_model::CreatePipelineLayoutError> {
3853 use crate::binding_model::CreatePipelineLayoutError as Error;
3854
3855 self.check_is_valid()?;
3856
3857 let bind_group_layouts_count = desc.bind_group_layouts.len();
3858 let device_max_bind_groups = self.limits.max_bind_groups as usize;
3859 if bind_group_layouts_count > device_max_bind_groups {
3860 return Err(Error::TooManyGroups {
3861 actual: bind_group_layouts_count,
3862 max: device_max_bind_groups,
3863 });
3864 }
3865
3866 if desc.immediate_size != 0 {
3867 self.require_features(wgt::Features::IMMEDIATES)?;
3868 }
3869 if self.limits.max_immediate_size < desc.immediate_size {
3870 return Err(Error::ImmediateRangeTooLarge {
3871 size: desc.immediate_size,
3872 max: self.limits.max_immediate_size,
3873 });
3874 }
3875 if !desc
3876 .immediate_size
3877 .is_multiple_of(wgt::IMMEDIATE_DATA_ALIGNMENT)
3878 {
3879 return Err(Error::MisalignedImmediateSize {
3880 size: desc.immediate_size,
3881 });
3882 }
3883
3884 let mut count_validator = binding_model::BindingTypeMaxCountValidator::default();
3885
3886 for (index, bgl) in desc.bind_group_layouts.iter().enumerate() {
3887 let Some(bgl) = bgl else {
3888 continue;
3889 };
3890
3891 bgl.same_device(self)?;
3892
3893 if !ignore_exclusive_pipeline_check {
3894 let exclusive_pipeline = bgl.exclusive_pipeline.get().unwrap();
3895 if !matches!(exclusive_pipeline, binding_model::ExclusivePipeline::None) {
3896 return Err(Error::BglHasExclusivePipeline {
3897 index,
3898 pipeline: alloc::format!("{exclusive_pipeline}"),
3899 });
3900 }
3901 }
3902
3903 count_validator.merge(&bgl.state()?.binding_count_validator);
3904 }
3905
3906 count_validator
3907 .validate(&self.limits, self.instance_flags)
3908 .map_err(Error::TooManyBindings)?;
3909
3910 let buffers_and_acceleration_structures_in_vertex_stage =
3911 count_validator.buffers_and_acceleration_structures_in_vertex_stage();
3912
3913 let get_bgl_iter = || {
3914 desc.bind_group_layouts
3915 .iter()
3916 .map(|bgl| bgl.as_ref().filter(|bgl| !bgl.entries.is_empty()))
3917 };
3918
3919 let bind_group_layouts = get_bgl_iter()
3920 .map(|bgl| bgl.cloned())
3921 .collect::<ArrayVec<_, { hal::MAX_BIND_GROUPS }>>();
3922
3923 let raw_bind_group_layouts = get_bgl_iter()
3924 .map(|bgl| bgl.map(|bgl| bgl.try_raw()).transpose())
3925 .collect::<Result<ArrayVec<_, { hal::MAX_BIND_GROUPS }>, _>>()?;
3926
3927 let additional_flags = if self.indirect_validation.is_some() {
3928 hal::PipelineLayoutFlags::INDIRECT_BUILTIN_UPDATE
3929 } else {
3930 hal::PipelineLayoutFlags::empty()
3931 };
3932
3933 let hal_desc = hal::PipelineLayoutDescriptor {
3934 label: desc.label.to_hal(self.instance_flags),
3935 flags: hal::PipelineLayoutFlags::FIRST_VERTEX_INSTANCE
3936 | hal::PipelineLayoutFlags::NUM_WORK_GROUPS
3937 | additional_flags,
3938 bind_group_layouts: &raw_bind_group_layouts,
3939 immediate_size: desc.immediate_size,
3940 };
3941
3942 let raw = unsafe { self.raw().create_pipeline_layout(&hal_desc) }
3943 .map_err(|e| self.handle_hal_error(e))?;
3944
3945 drop(raw_bind_group_layouts);
3946
3947 let layout = binding_model::PipelineLayout {
3948 raw: ResourceState::Valid(raw),
3949 device: self.clone(),
3950 label: desc.label.to_string(),
3951 bind_group_layouts,
3952 immediate_size: desc.immediate_size,
3953 buffers_and_acceleration_structures_in_vertex_stage,
3954 };
3955
3956 let layout = Arc::new(layout);
3957
3958 Ok(layout)
3959 }
3960
3961 fn create_derived_pipeline_layout(
3962 self: &Arc<Self>,
3963 mut derived_group_layouts: Box<ArrayVec<bgl::EntryMap, { hal::MAX_BIND_GROUPS }>>,
3964 immediate_size: u32,
3965 ) -> Result<Arc<binding_model::PipelineLayout>, pipeline::ImplicitLayoutError> {
3966 while derived_group_layouts
3967 .last()
3968 .is_some_and(|map| map.is_empty())
3969 {
3970 derived_group_layouts.pop();
3971 }
3972
3973 let mut unique_bind_group_layouts = FastHashMap::default();
3974
3975 let bind_group_layouts = derived_group_layouts
3976 .into_iter()
3977 .map(|mut bgl_entry_map| {
3978 if bgl_entry_map.is_empty() {
3979 return Ok(None);
3980 }
3981
3982 bgl_entry_map.sort();
3983 match unique_bind_group_layouts.entry(bgl_entry_map) {
3984 hashbrown::hash_map::Entry::Occupied(v) => Ok(Some(Arc::clone(v.get()))),
3985 hashbrown::hash_map::Entry::Vacant(e) => {
3986 match self.create_bind_group_layout_impl(
3987 &None,
3988 e.key().clone(),
3989 bgl::Origin::Derived,
3990 ) {
3991 Ok(bgl) => {
3992 e.insert(bgl.clone());
3993 Ok(Some(bgl))
3994 }
3995 Err(e) => Err(e),
3996 }
3997 }
3998 }
3999 })
4000 .collect::<Result<Vec<_>, _>>()?;
4001
4002 let layout_desc = binding_model::ResolvedPipelineLayoutDescriptor {
4003 label: None,
4004 bind_group_layouts: Cow::Owned(bind_group_layouts),
4005 immediate_size,
4006 };
4007
4008 let layout = self.create_pipeline_layout_impl(&layout_desc, true)?;
4009 Ok(layout)
4010 }
4011
4012 pub fn create_compute_pipeline(
4013 self: &Arc<Self>,
4014 desc: pipeline::ResolvedComputePipelineDescriptor,
4015 ) -> (
4016 Arc<pipeline::ComputePipeline>,
4017 Option<pipeline::CreateComputePipelineError>,
4018 ) {
4019 let (compute_pipeline, error) = match self.create_compute_pipeline_inner(desc.clone()) {
4020 Ok(compute_pipeline) => (compute_pipeline, None),
4021 Err(error) => (
4022 pipeline::ComputePipeline::invalid(self.clone(), desc.label.to_string()),
4023 Some(error),
4024 ),
4025 };
4026 #[cfg(feature = "trace")]
4027 if let Some(ref mut trace) = *self.trace.lock() {
4028 use crate::device::trace;
4029 use crate::device::trace::IntoTrace;
4030 trace.add(trace::Action::CreateComputePipeline {
4031 id: compute_pipeline.to_trace(),
4032 desc: desc.to_trace(),
4033 });
4034 }
4035 (compute_pipeline, error)
4036 }
4037
4038 pub fn create_compute_pipeline_inner(
4039 self: &Arc<Self>,
4040 desc: pipeline::ResolvedComputePipelineDescriptor,
4041 ) -> Result<Arc<pipeline::ComputePipeline>, pipeline::CreateComputePipelineError> {
4042 self.check_is_valid()?;
4043
4044 self.require_downlevel_flags(wgt::DownlevelFlags::COMPUTE_SHADERS)?;
4045
4046 let shader_module = desc.stage.module;
4047
4048 shader_module.same_device(self)?;
4049
4050 let is_auto_layout = desc.layout.is_none();
4051
4052 let pipeline_layout = match desc.layout {
4054 Some(pipeline_layout) => {
4055 pipeline_layout.same_device(self)?;
4056 pipeline_layout.check_valid()?;
4057 Some(pipeline_layout)
4058 }
4059 None => None,
4060 };
4061
4062 let mut binding_layout_source = match pipeline_layout {
4063 Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4064 None => validation::BindingLayoutSource::new_derived(&self.limits),
4065 };
4066 let mut shader_binding_sizes = FastHashMap::default();
4067 let io = validation::StageIo::default();
4068
4069 let final_entry_point_name;
4070
4071 {
4072 let stage = validation::ShaderStageForValidation::Compute;
4073
4074 final_entry_point_name = shader_module.finalize_entry_point_name(
4075 stage.to_naga(),
4076 desc.stage.entry_point.as_ref().map(|ep| ep.as_ref()),
4077 )?;
4078
4079 if let Some(interface) = shader_module.interface.interface() {
4080 let _ = interface.check_stage(
4081 &mut binding_layout_source,
4082 &mut shader_binding_sizes,
4083 &final_entry_point_name,
4084 stage,
4085 io,
4086 None,
4087 )?;
4088 }
4089 }
4090
4091 let pipeline_layout = match binding_layout_source {
4092 validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4093 validation::BindingLayoutSource::Derived(entries) => {
4094 let immediate_size = shader_module
4095 .interface
4096 .interface()
4097 .map_or(0, |i| i.immediate_size);
4098 self.create_derived_pipeline_layout(entries, immediate_size)?
4099 }
4100 };
4101
4102 let late_sized_buffer_groups =
4103 Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4104
4105 let cache = match desc.cache {
4106 Some(cache) => {
4107 cache.same_device(self)?;
4108 Some(cache)
4109 }
4110 None => None,
4111 };
4112
4113 let pipeline_desc = hal::ComputePipelineDescriptor {
4114 label: desc.label.to_hal(self.instance_flags),
4115 layout: pipeline_layout.raw()?,
4116 stage: hal::ProgrammableStage {
4117 module: shader_module.raw(),
4118 entry_point: final_entry_point_name.as_ref(),
4119 constants: &desc.stage.constants,
4120 zero_initialize_workgroup_memory: desc.stage.zero_initialize_workgroup_memory,
4121 },
4122 cache: cache.as_ref().map(|it| it.raw()),
4123 };
4124
4125 let raw =
4126 unsafe { self.raw().create_compute_pipeline(&pipeline_desc) }.map_err(
4127 |err| match err {
4128 hal::PipelineError::Device(error) => {
4129 pipeline::CreateComputePipelineError::Device(self.handle_hal_error(error))
4130 }
4131 hal::PipelineError::Linkage(_stages, msg) => {
4132 pipeline::CreateComputePipelineError::Internal(msg)
4133 }
4134 hal::PipelineError::EntryPoint(_stage) => {
4135 pipeline::CreateComputePipelineError::Internal(
4136 ENTRYPOINT_FAILURE_ERROR.to_string(),
4137 )
4138 }
4139 hal::PipelineError::PipelineConstants(_stages, msg) => {
4140 pipeline::CreateComputePipelineError::PipelineConstants(msg)
4141 }
4142 },
4143 )?;
4144
4145 let immediate_slots_required =
4146 shader_module
4147 .interface
4148 .interface()
4149 .map_or(Default::default(), |iface| {
4150 iface.immediate_slots_required(
4151 naga::ShaderStage::Compute,
4152 &final_entry_point_name,
4153 )
4154 });
4155
4156 let pipeline = pipeline::ComputePipeline {
4157 state: ResourceState::Valid(pipeline::ComputePipelineState {
4158 raw: ManuallyDrop::new(raw),
4159 layout: pipeline_layout.clone(),
4160 _shader_module: shader_module,
4161 }),
4162 device: self.clone(),
4163 late_sized_buffer_groups,
4164 immediate_slots_required,
4165 label: desc.label.to_string(),
4166 tracking_data: TrackingData::new(self.tracker_indices.compute_pipelines.clone()),
4167 };
4168
4169 let pipeline = Arc::new(pipeline);
4170
4171 if is_auto_layout {
4172 for bgl in pipeline_layout.bind_group_layouts.iter() {
4173 let Some(bgl) = bgl else {
4174 continue;
4175 };
4176
4177 let _ = bgl.exclusive_pipeline.set((&pipeline).into());
4180 }
4181 }
4182
4183 Ok(pipeline)
4184 }
4185
4186 pub fn create_render_pipeline(
4187 self: &Arc<Self>,
4188 desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4189 ) -> (
4190 Arc<pipeline::RenderPipeline>,
4191 Option<pipeline::CreateRenderPipelineError>,
4192 ) {
4193 let (render_pipeline, error) = match self.create_render_pipeline_inner(desc.clone()) {
4194 Ok(pipeline) => (pipeline, None),
4195 Err(e) => (
4196 pipeline::RenderPipeline::invalid(self.clone(), desc.label.to_string()),
4197 Some(e),
4198 ),
4199 };
4200 #[cfg(feature = "trace")]
4201 if let Some(ref mut trace) = *self.trace.lock() {
4202 use crate::device::trace::IntoTrace;
4203 trace.add(trace::Action::CreateGeneralRenderPipeline {
4204 id: render_pipeline.to_trace(),
4205 desc: desc.to_trace(),
4206 });
4207 }
4208 (render_pipeline, error)
4209 }
4210
4211 pub fn create_render_pipeline_inner(
4212 self: &Arc<Self>,
4213 desc: pipeline::ResolvedGeneralRenderPipelineDescriptor,
4214 ) -> Result<Arc<pipeline::RenderPipeline>, pipeline::CreateRenderPipelineError> {
4215 use wgt::TextureFormatFeatureFlags as Tfff;
4216
4217 self.check_is_valid()?;
4218
4219 let mut shader_binding_sizes = FastHashMap::default();
4220
4221 let color_targets = desc
4222 .fragment
4223 .as_ref()
4224 .map_or(&[][..], |fragment| &fragment.targets);
4225 let depth_stencil_state = desc.depth_stencil.as_ref();
4226
4227 check_color_attachment_count(color_targets.len(), self.limits.max_color_attachments)?;
4228
4229 {
4230 let cts: ArrayVec<_, { hal::MAX_COLOR_ATTACHMENTS }> =
4231 color_targets.iter().filter_map(|x| x.as_ref()).collect();
4232 if !cts.is_empty() && {
4233 let first = &cts[0];
4234 cts[1..]
4235 .iter()
4236 .any(|ct| ct.write_mask != first.write_mask || ct.blend != first.blend)
4237 } {
4238 self.require_downlevel_flags(wgt::DownlevelFlags::INDEPENDENT_BLEND)?;
4239 }
4240 }
4241
4242 let mut io = validation::StageIo::default();
4243 let mut validated_stages = wgt::ShaderStages::empty();
4244
4245 let mut vertex_steps;
4246 let mut hal_vertex_buffer_layouts;
4247 let mut total_attributes;
4248 let mut dual_source_blending = false;
4249 let mut has_depth_attachment = false;
4250 if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4251 if vertex.buffers.len() > self.limits.max_vertex_buffers as usize {
4252 return Err(pipeline::CreateRenderPipelineError::TooManyVertexBuffers {
4253 given: vertex.buffers.len() as u32,
4254 limit: self.limits.max_vertex_buffers,
4255 });
4256 }
4257
4258 vertex_steps = Vec::with_capacity(vertex.buffers.len());
4259 hal_vertex_buffer_layouts = Vec::with_capacity(vertex.buffers.len());
4260 total_attributes = 0;
4261 for (i, vb_state) in vertex.buffers.iter().enumerate() {
4262 let Some(vb_state) = vb_state else {
4263 vertex_steps.push(None);
4264 hal_vertex_buffer_layouts.push(None);
4265 continue;
4266 };
4267
4268 if vb_state.array_stride > self.limits.max_vertex_buffer_array_stride as u64 {
4271 return Err(pipeline::CreateRenderPipelineError::VertexStrideTooLarge {
4272 index: i as u32,
4273 given: vb_state.array_stride as u32,
4274 limit: self.limits.max_vertex_buffer_array_stride,
4275 });
4276 }
4277 if vb_state.array_stride % wgt::VERTEX_ALIGNMENT != 0 {
4278 return Err(pipeline::CreateRenderPipelineError::UnalignedVertexStride {
4279 index: i as u32,
4280 stride: vb_state.array_stride,
4281 });
4282 }
4283
4284 let max_stride = if vb_state.array_stride == 0 {
4285 self.limits.max_vertex_buffer_array_stride as u64
4286 } else {
4287 vb_state.array_stride
4288 };
4289 let mut last_stride = 0;
4290 for attribute in vb_state.attributes.iter() {
4291 let attribute_stride = attribute.offset + attribute.format.size();
4292 if attribute_stride > max_stride {
4293 return Err(
4294 pipeline::CreateRenderPipelineError::VertexAttributeStrideTooLarge {
4295 location: attribute.shader_location,
4296 given: attribute_stride as u32,
4297 limit: max_stride as u32,
4298 },
4299 );
4300 }
4301
4302 let required_offset_alignment = attribute.format.size().min(4);
4303 if attribute.offset % required_offset_alignment != 0 {
4304 return Err(
4305 pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4306 location: attribute.shader_location,
4307 offset: attribute.offset,
4308 },
4309 );
4310 }
4311
4312 if attribute.shader_location >= self.limits.max_vertex_attributes {
4313 return Err(
4314 pipeline::CreateRenderPipelineError::VertexAttributeLocationTooLarge {
4315 given: attribute.shader_location,
4316 limit: self.limits.max_vertex_attributes,
4317 },
4318 );
4319 }
4320
4321 last_stride = last_stride.max(attribute_stride);
4322 }
4323
4324 vertex_steps.push(Some(pipeline::VertexStep {
4325 stride: vb_state.array_stride,
4326 last_stride,
4327 mode: vb_state.step_mode,
4328 }));
4329 hal_vertex_buffer_layouts.push(if vb_state.attributes.is_empty() {
4330 None
4331 } else {
4332 Some(hal::VertexBufferLayout {
4333 array_stride: vb_state.array_stride,
4334 step_mode: vb_state.step_mode,
4335 attributes: vb_state.attributes.as_ref(),
4336 })
4337 });
4338
4339 for attribute in vb_state.attributes.iter() {
4340 if attribute.offset >= 0x10000000 {
4341 return Err(
4342 pipeline::CreateRenderPipelineError::InvalidVertexAttributeOffset {
4343 location: attribute.shader_location,
4344 offset: attribute.offset,
4345 },
4346 );
4347 }
4348
4349 if let wgt::VertexFormat::Float64
4350 | wgt::VertexFormat::Float64x2
4351 | wgt::VertexFormat::Float64x3
4352 | wgt::VertexFormat::Float64x4 = attribute.format
4353 {
4354 self.require_features(wgt::Features::VERTEX_ATTRIBUTE_64BIT)?;
4355 }
4356
4357 let previous = io.varyings.insert(
4358 attribute.shader_location,
4359 validation::InterfaceVar::vertex_attribute(attribute.format),
4360 );
4361
4362 if previous.is_some() {
4363 return Err(pipeline::CreateRenderPipelineError::ShaderLocationClash(
4364 attribute.shader_location,
4365 ));
4366 }
4367 }
4368 total_attributes += vb_state.attributes.len();
4369 }
4370
4371 if total_attributes > self.limits.max_vertex_attributes as usize {
4372 return Err(
4373 pipeline::CreateRenderPipelineError::TooManyVertexAttributes {
4374 given: total_attributes as u32,
4375 limit: self.limits.max_vertex_attributes,
4376 },
4377 );
4378 }
4379 } else {
4380 vertex_steps = Vec::new();
4381 hal_vertex_buffer_layouts = Vec::new();
4382 };
4383
4384 if desc.primitive.strip_index_format.is_some() && !desc.primitive.topology.is_strip() {
4385 return Err(
4386 pipeline::CreateRenderPipelineError::StripIndexFormatForNonStripTopology {
4387 strip_index_format: desc.primitive.strip_index_format,
4388 topology: desc.primitive.topology,
4389 },
4390 );
4391 }
4392
4393 if desc.primitive.unclipped_depth {
4394 self.require_features(wgt::Features::DEPTH_CLIP_CONTROL)?;
4395 }
4396
4397 if desc.primitive.polygon_mode == wgt::PolygonMode::Line {
4398 self.require_features(wgt::Features::POLYGON_MODE_LINE)?;
4399 }
4400 if desc.primitive.polygon_mode == wgt::PolygonMode::Point {
4401 self.require_features(wgt::Features::POLYGON_MODE_POINT)?;
4402 }
4403
4404 if desc.primitive.conservative {
4405 self.require_features(wgt::Features::CONSERVATIVE_RASTERIZATION)?;
4406 }
4407
4408 if desc.primitive.conservative && desc.primitive.polygon_mode != wgt::PolygonMode::Fill {
4409 return Err(
4410 pipeline::CreateRenderPipelineError::ConservativeRasterizationNonFillPolygonMode,
4411 );
4412 }
4413
4414 let mut target_specified = false;
4415
4416 for (i, cs) in color_targets.iter().enumerate() {
4417 if let Some(cs) = cs.as_ref() {
4418 target_specified = true;
4419 let error = 'error: {
4420 if cs.write_mask.contains_unknown_bits() {
4424 break 'error Some(ColorStateError::InvalidWriteMask(cs.write_mask));
4425 }
4426
4427 let format_features = self.describe_format_features(cs.format)?;
4428 if !format_features
4429 .allowed_usages
4430 .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4431 {
4432 break 'error Some(ColorStateError::FormatNotRenderable(cs.format));
4433 }
4434 if cs.blend.is_some() && !format_features.flags.contains(Tfff::BLENDABLE) {
4435 break 'error Some(ColorStateError::FormatNotBlendable(cs.format));
4436 }
4437 if !hal::FormatAspects::from(cs.format).contains(hal::FormatAspects::COLOR) {
4438 break 'error Some(ColorStateError::FormatNotColor(cs.format));
4439 }
4440
4441 if desc.multisample.count > 1
4442 && !format_features
4443 .flags
4444 .sample_count_supported(desc.multisample.count)
4445 {
4446 break 'error Some(ColorStateError::InvalidSampleCount(
4447 desc.multisample.count,
4448 cs.format,
4449 cs.format
4450 .guaranteed_format_features(self.features)
4451 .flags
4452 .supported_sample_counts(),
4453 self.adapter
4454 .get_texture_format_features(cs.format)
4455 .flags
4456 .supported_sample_counts(),
4457 ));
4458 }
4459
4460 if let Some(blend_mode) = cs.blend {
4461 for component in [&blend_mode.color, &blend_mode.alpha] {
4462 for factor in [component.src_factor, component.dst_factor] {
4463 if factor.ref_second_blend_source() {
4464 self.require_features(wgt::Features::DUAL_SOURCE_BLENDING)?;
4465 if i == 0 {
4466 dual_source_blending = true;
4467 } else {
4468 break 'error Some(
4469 ColorStateError::BlendFactorOnUnsupportedTarget {
4470 factor,
4471 target: i as u32,
4472 },
4473 );
4474 }
4475 }
4476
4477 if [wgt::BlendOperation::Min, wgt::BlendOperation::Max]
4478 .contains(&component.operation)
4479 && factor != wgt::BlendFactor::One
4480 {
4481 break 'error Some(ColorStateError::InvalidMinMaxBlendFactor {
4482 factor,
4483 target: i as u32,
4484 });
4485 }
4486 }
4487 }
4488 }
4489
4490 break 'error None;
4491 };
4492 if let Some(e) = error {
4493 return Err(pipeline::CreateRenderPipelineError::ColorState(i as u8, e));
4494 }
4495 }
4496 }
4497
4498 if dual_source_blending && color_targets.len() > 1 {
4499 return Err(
4500 pipeline::CreateRenderPipelineError::DualSourceBlendingWithMultipleColorTargets {
4501 count: color_targets.len(),
4502 },
4503 );
4504 }
4505
4506 validation::validate_color_attachment_bytes_per_sample(
4507 color_targets.iter().flatten().map(|cs| cs.format),
4508 self.limits.max_color_attachment_bytes_per_sample,
4509 )
4510 .map_err(pipeline::CreateRenderPipelineError::ColorAttachment)?;
4511
4512 if let Some(ds) = depth_stencil_state {
4513 target_specified = true;
4515 let error = 'error: {
4516 if !ds.format.is_depth_stencil_format() {
4517 break 'error Some(pipeline::DepthStencilStateError::FormatNotDepthOrStencil(
4520 ds.format,
4521 ));
4522 }
4523
4524 let format_features = self.describe_format_features(ds.format)?;
4525 if !format_features
4526 .allowed_usages
4527 .contains(wgt::TextureUsages::RENDER_ATTACHMENT)
4528 {
4529 break 'error Some(pipeline::DepthStencilStateError::FormatNotRenderable(
4530 ds.format,
4531 ));
4532 }
4533
4534 let aspect = hal::FormatAspects::from(ds.format);
4535 if aspect.contains(hal::FormatAspects::DEPTH) {
4536 has_depth_attachment = true;
4537 } else if ds.is_depth_enabled() {
4538 break 'error Some(pipeline::DepthStencilStateError::FormatNotDepth(ds.format));
4539 }
4540 if has_depth_attachment {
4541 let Some(depth_write_enabled) = ds.depth_write_enabled else {
4542 break 'error Some(
4543 pipeline::DepthStencilStateError::MissingDepthWriteEnabled(ds.format),
4544 );
4545 };
4546
4547 let depth_compare_required = depth_write_enabled
4548 || ds.stencil.front.depth_fail_op != wgt::StencilOperation::Keep
4549 || ds.stencil.back.depth_fail_op != wgt::StencilOperation::Keep;
4550 if depth_compare_required && ds.depth_compare.is_none() {
4551 break 'error Some(pipeline::DepthStencilStateError::MissingDepthCompare(
4552 ds.format,
4553 ));
4554 }
4555 }
4556
4557 if ds.stencil.is_enabled() && !aspect.contains(hal::FormatAspects::STENCIL) {
4558 break 'error Some(pipeline::DepthStencilStateError::FormatNotStencil(
4559 ds.format,
4560 ));
4561 }
4562 if desc.multisample.count > 1
4563 && !format_features
4564 .flags
4565 .sample_count_supported(desc.multisample.count)
4566 {
4567 break 'error Some(pipeline::DepthStencilStateError::InvalidSampleCount(
4568 desc.multisample.count,
4569 ds.format,
4570 ds.format
4571 .guaranteed_format_features(self.features)
4572 .flags
4573 .supported_sample_counts(),
4574 self.adapter
4575 .get_texture_format_features(ds.format)
4576 .flags
4577 .supported_sample_counts(),
4578 ));
4579 }
4580
4581 break 'error None;
4582 };
4583 if let Some(e) = error {
4584 return Err(pipeline::CreateRenderPipelineError::DepthStencilState(e));
4585 }
4586
4587 if ds.bias.clamp != 0.0 {
4588 self.require_downlevel_flags(wgt::DownlevelFlags::DEPTH_BIAS_CLAMP)?;
4589 }
4590
4591 if (ds.bias.is_enabled() || ds.bias.clamp != 0.0)
4592 && !desc.primitive.topology.is_triangles()
4593 {
4594 return Err(pipeline::CreateRenderPipelineError::DepthStencilState(
4595 pipeline::DepthStencilStateError::DepthBiasWithIncompatibleTopology(
4596 desc.primitive.topology,
4597 ),
4598 ));
4599 }
4600 }
4601
4602 if !target_specified {
4603 return Err(pipeline::CreateRenderPipelineError::NoTargetSpecified);
4604 }
4605
4606 let is_auto_layout = desc.layout.is_none();
4607
4608 let pipeline_layout = match desc.layout {
4610 Some(pipeline_layout) => {
4611 pipeline_layout.same_device(self)?;
4612 pipeline_layout.check_valid()?;
4613 Some(pipeline_layout)
4614 }
4615 None => None,
4616 };
4617
4618 let mut binding_layout_source = match pipeline_layout {
4619 Some(pipeline_layout) => validation::BindingLayoutSource::Provided(pipeline_layout),
4620 None => validation::BindingLayoutSource::new_derived(&self.limits),
4621 };
4622
4623 let samples = {
4624 let sc = desc.multisample.count;
4625 if sc == 0 || sc > 32 || !sc.is_power_of_two() {
4626 return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc));
4627 }
4628 sc
4629 };
4630
4631 let mut vertex_stage = None;
4632 let mut task_stage = None;
4633 let mut mesh_stage = None;
4634 let mut _vertex_entry_point_name = String::new();
4635 let mut _task_entry_point_name = String::new();
4636 let mut _mesh_entry_point_name = String::new();
4637 let mut immediate_slots_required = naga::valid::ImmediateSlots::default();
4638 match desc.vertex {
4639 pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) => {
4640 vertex_stage = {
4641 let stage_desc = &vertex.stage;
4642 let stage = validation::ShaderStageForValidation::Vertex {
4643 topology: desc.primitive.topology,
4644 compare_function: desc.depth_stencil.as_ref().and_then(|d| d.depth_compare),
4645 };
4646 let stage_bit = stage.to_wgt_bit();
4647
4648 let vertex_shader_module = &stage_desc.module;
4649 vertex_shader_module.same_device(self)?;
4650
4651 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4652 stage: stage_bit,
4653 error,
4654 };
4655
4656 _vertex_entry_point_name = vertex_shader_module
4657 .finalize_entry_point_name(
4658 stage.to_naga(),
4659 stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4660 )
4661 .map_err(stage_err)?;
4662
4663 if let Some(interface) = vertex_shader_module.interface.interface() {
4664 immediate_slots_required |= interface
4665 .immediate_slots_required(stage.to_naga(), &_vertex_entry_point_name);
4666 io = interface
4667 .check_stage(
4668 &mut binding_layout_source,
4669 &mut shader_binding_sizes,
4670 &_vertex_entry_point_name,
4671 stage,
4672 io,
4673 Some(desc.primitive.topology),
4674 )
4675 .map_err(stage_err)?;
4676 validated_stages |= stage_bit;
4677 }
4678 Some(hal::ProgrammableStage {
4679 module: vertex_shader_module.raw(),
4680 entry_point: &_vertex_entry_point_name,
4681 constants: &stage_desc.constants,
4682 zero_initialize_workgroup_memory: stage_desc
4683 .zero_initialize_workgroup_memory,
4684 })
4685 };
4686 }
4687 pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => {
4688 self.require_features(wgt::Features::EXPERIMENTAL_MESH_SHADER)?;
4689
4690 task_stage = if let Some(task) = task {
4691 let stage_desc = &task.stage;
4692 let stage = validation::ShaderStageForValidation::Task;
4693 let stage_bit = stage.to_wgt_bit();
4694 let task_shader_module = &stage_desc.module;
4695 task_shader_module.same_device(self)?;
4696
4697 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4698 stage: stage_bit,
4699 error,
4700 };
4701
4702 _task_entry_point_name = task_shader_module
4703 .finalize_entry_point_name(
4704 stage.to_naga(),
4705 stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4706 )
4707 .map_err(stage_err)?;
4708
4709 if let Some(interface) = task_shader_module.interface.interface() {
4710 immediate_slots_required |= interface
4711 .immediate_slots_required(stage.to_naga(), &_task_entry_point_name);
4712 io = interface
4713 .check_stage(
4714 &mut binding_layout_source,
4715 &mut shader_binding_sizes,
4716 &_task_entry_point_name,
4717 stage,
4718 io,
4719 Some(desc.primitive.topology),
4720 )
4721 .map_err(stage_err)?;
4722 validated_stages |= stage_bit;
4723 }
4724 Some(hal::ProgrammableStage {
4725 module: task_shader_module.raw(),
4726 entry_point: &_task_entry_point_name,
4727 constants: &stage_desc.constants,
4728 zero_initialize_workgroup_memory: stage_desc
4729 .zero_initialize_workgroup_memory,
4730 })
4731 } else {
4732 None
4733 };
4734 mesh_stage = {
4735 let stage_desc = &mesh.stage;
4736 let stage = validation::ShaderStageForValidation::Mesh;
4737 let stage_bit = stage.to_wgt_bit();
4738 let mesh_shader_module = &stage_desc.module;
4739 mesh_shader_module.same_device(self)?;
4740
4741 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4742 stage: stage_bit,
4743 error,
4744 };
4745
4746 _mesh_entry_point_name = mesh_shader_module
4747 .finalize_entry_point_name(
4748 stage.to_naga(),
4749 stage_desc.entry_point.as_ref().map(|ep| ep.as_ref()),
4750 )
4751 .map_err(stage_err)?;
4752
4753 if let Some(interface) = mesh_shader_module.interface.interface() {
4754 immediate_slots_required |= interface
4755 .immediate_slots_required(stage.to_naga(), &_mesh_entry_point_name);
4756 io = interface
4757 .check_stage(
4758 &mut binding_layout_source,
4759 &mut shader_binding_sizes,
4760 &_mesh_entry_point_name,
4761 stage,
4762 io,
4763 Some(desc.primitive.topology),
4764 )
4765 .map_err(stage_err)?;
4766 validated_stages |= stage_bit;
4767 }
4768 Some(hal::ProgrammableStage {
4769 module: mesh_shader_module.raw(),
4770 entry_point: &_mesh_entry_point_name,
4771 constants: &stage_desc.constants,
4772 zero_initialize_workgroup_memory: stage_desc
4773 .zero_initialize_workgroup_memory,
4774 })
4775 };
4776 }
4777 }
4778
4779 let fragment_entry_point_name;
4780 let fragment_stage = match desc.fragment {
4781 Some(ref fragment_state) => {
4782 let stage = validation::ShaderStageForValidation::Fragment {
4783 dual_source_blending,
4784 has_depth_attachment,
4785 };
4786 let stage_bit = stage.to_wgt_bit();
4787
4788 let shader_module = &fragment_state.stage.module;
4789 shader_module.same_device(self)?;
4790
4791 let stage_err = |error| pipeline::CreateRenderPipelineError::Stage {
4792 stage: stage_bit,
4793 error,
4794 };
4795
4796 fragment_entry_point_name = shader_module
4797 .finalize_entry_point_name(
4798 stage.to_naga(),
4799 fragment_state
4800 .stage
4801 .entry_point
4802 .as_ref()
4803 .map(|ep| ep.as_ref()),
4804 )
4805 .map_err(stage_err)?;
4806
4807 if let Some(interface) = shader_module.interface.interface() {
4808 immediate_slots_required |= interface
4809 .immediate_slots_required(stage.to_naga(), &fragment_entry_point_name);
4810 io = interface
4811 .check_stage(
4812 &mut binding_layout_source,
4813 &mut shader_binding_sizes,
4814 &fragment_entry_point_name,
4815 stage,
4816 io,
4817 Some(desc.primitive.topology),
4818 )
4819 .map_err(stage_err)?;
4820 validated_stages |= stage_bit;
4821 }
4822
4823 Some(hal::ProgrammableStage {
4824 module: shader_module.raw(),
4825 entry_point: &fragment_entry_point_name,
4826 constants: &fragment_state.stage.constants,
4827 zero_initialize_workgroup_memory: fragment_state
4828 .stage
4829 .zero_initialize_workgroup_memory,
4830 })
4831 }
4832 None => None,
4833 };
4834
4835 if validated_stages.contains(wgt::ShaderStages::FRAGMENT) {
4836 for (i, output) in io.varyings.iter() {
4837 match color_targets.get(*i as usize) {
4838 Some(Some(state)) => {
4839 validation::check_texture_format(state.format, &output.ty).map_err(
4840 |pipeline| {
4841 pipeline::CreateRenderPipelineError::ColorState(
4842 *i as u8,
4843 ColorStateError::IncompatibleFormat {
4844 pipeline,
4845 shader: output.ty,
4846 },
4847 )
4848 },
4849 )?;
4850 }
4851 _ => {
4852 log::debug!(
4853 "The fragment stage {:?} output @location({}) values are ignored",
4854 fragment_stage
4855 .as_ref()
4856 .map_or("", |stage| stage.entry_point),
4857 i
4858 );
4859 }
4860 }
4861 }
4862 }
4863 let last_stage = match desc.fragment {
4864 Some(_) => wgt::ShaderStages::FRAGMENT,
4865 None => wgt::ShaderStages::VERTEX,
4866 };
4867 if is_auto_layout && !validated_stages.contains(last_stage) {
4868 return Err(pipeline::ImplicitLayoutError::ReflectionError(last_stage).into());
4869 }
4870
4871 let pipeline_layout = match binding_layout_source {
4872 validation::BindingLayoutSource::Provided(pipeline_layout) => pipeline_layout,
4873 validation::BindingLayoutSource::Derived(entries) => {
4874 let immediate_size = {
4875 let immediate_size_of = |sm: &pipeline::ShaderModule| {
4876 sm.interface.interface().map(|i| i.immediate_size)
4877 };
4878 let vertex = match desc.vertex {
4879 pipeline::RenderPipelineVertexProcessor::Vertex(ref v) => {
4880 immediate_size_of(&v.stage.module)
4881 }
4882 pipeline::RenderPipelineVertexProcessor::Mesh(ref task, ref mesh) => task
4883 .as_ref()
4884 .and_then(|t| immediate_size_of(&t.stage.module))
4885 .max(immediate_size_of(&mesh.stage.module)),
4886 };
4887 let fragment = desc
4888 .fragment
4889 .as_ref()
4890 .and_then(|f| immediate_size_of(&f.stage.module));
4891 vertex.max(fragment).unwrap_or(0)
4892 };
4893 self.create_derived_pipeline_layout(entries, immediate_size)?
4894 }
4895 };
4896
4897 if let pipeline::RenderPipelineVertexProcessor::Vertex(ref vertex) = desc.vertex {
4898 let bind_groups_plus_vertex_buffers =
4899 u32::try_from(pipeline_layout.bind_group_layouts.len() + vertex.buffers.len())
4900 .unwrap();
4901 if bind_groups_plus_vertex_buffers > self.limits.max_bind_groups_plus_vertex_buffers {
4902 return Err(
4903 pipeline::CreateRenderPipelineError::TooManyBindGroupsPlusVertexBuffers {
4904 given: bind_groups_plus_vertex_buffers,
4905 limit: self.limits.max_bind_groups_plus_vertex_buffers,
4906 },
4907 );
4908 }
4909
4910 let given = pipeline_layout
4911 .buffers_and_acceleration_structures_in_vertex_stage
4912 .saturating_add(vertex.buffers.len() as u32);
4913 if !self
4914 .instance_flags
4915 .contains(wgt::InstanceFlags::STRICT_WEBGPU_COMPLIANCE)
4916 {
4917 let limit = self
4918 .limits
4919 .max_buffers_and_acceleration_structures_per_shader_stage;
4920 if given > limit {
4921 return Err(
4922 pipeline::CreateRenderPipelineError::TooManyBuffersAndAccelerationStructuresInVertexStage {
4923 given,
4924 limit,
4925 },
4926 );
4927 }
4928 }
4929 }
4930
4931 if let Some(mv_mask) = desc.multiview_mask {
4933 self.require_features(wgt::Features::MULTIVIEW)?;
4934 if !(mv_mask.get() + 1).is_power_of_two() {
4935 self.require_features(wgt::Features::SELECTIVE_MULTIVIEW)?;
4936 }
4937 }
4938
4939 if !self
4940 .downlevel
4941 .flags
4942 .contains(wgt::DownlevelFlags::BUFFER_BINDINGS_NOT_16_BYTE_ALIGNED)
4943 {
4944 for (binding, size) in shader_binding_sizes.iter() {
4945 if size.get() % 16 != 0 {
4946 return Err(pipeline::CreateRenderPipelineError::UnalignedShader {
4947 binding: binding.binding,
4948 group: binding.group,
4949 size: size.get(),
4950 });
4951 }
4952 }
4953 }
4954
4955 let late_sized_buffer_groups =
4956 Device::make_late_sized_buffer_groups(&shader_binding_sizes, &pipeline_layout);
4957
4958 let cache = match desc.cache {
4959 Some(cache) => {
4960 cache.same_device(self)?;
4961 Some(cache)
4962 }
4963 None => None,
4964 };
4965
4966 let is_mesh = mesh_stage.is_some();
4967 let has_task_shader = task_stage.is_some();
4968 let raw = {
4969 let pipeline_desc = hal::RenderPipelineDescriptor {
4970 label: desc.label.to_hal(self.instance_flags),
4971 layout: pipeline_layout.raw()?,
4972 vertex_processor: match vertex_stage {
4973 Some(vertex_stage) => hal::VertexProcessor::Standard {
4974 vertex_buffers: &hal_vertex_buffer_layouts,
4975 vertex_stage,
4976 },
4977 None => hal::VertexProcessor::Mesh {
4978 task_stage,
4979 mesh_stage: mesh_stage.unwrap(),
4980 },
4981 },
4982 primitive: desc.primitive,
4983 depth_stencil: desc.depth_stencil.clone(),
4984 multisample: desc.multisample,
4985 fragment_stage,
4986 color_targets,
4987 multiview_mask: desc.multiview_mask,
4988 cache: cache.as_ref().map(|it| it.raw()),
4989 };
4990 unsafe { self.raw().create_render_pipeline(&pipeline_desc) }.map_err(
4991 |err| match err {
4992 hal::PipelineError::Device(error) => {
4993 pipeline::CreateRenderPipelineError::Device(self.handle_hal_error(error))
4994 }
4995 hal::PipelineError::Linkage(stage, msg) => {
4996 pipeline::CreateRenderPipelineError::Internal { stage, error: msg }
4997 }
4998 hal::PipelineError::EntryPoint(stage) => {
4999 pipeline::CreateRenderPipelineError::Internal {
5000 stage: hal::auxil::map_naga_stage(stage),
5001 error: ENTRYPOINT_FAILURE_ERROR.to_string(),
5002 }
5003 }
5004 hal::PipelineError::PipelineConstants(stage, error) => {
5005 pipeline::CreateRenderPipelineError::PipelineConstants { stage, error }
5006 }
5007 },
5008 )?
5009 };
5010
5011 let pass_context = RenderPassContext {
5012 attachments: AttachmentData {
5013 colors: color_targets
5014 .iter()
5015 .map(|state| state.as_ref().map(|s| s.format))
5016 .collect(),
5017 resolves: ArrayVec::new(),
5018 depth_stencil: depth_stencil_state.as_ref().map(|state| state.format),
5019 },
5020 sample_count: samples,
5021 multiview_mask: desc.multiview_mask,
5022 };
5023
5024 let mut flags = pipeline::PipelineFlags::empty();
5025 for state in color_targets.iter().filter_map(|s| s.as_ref()) {
5026 if let Some(ref bs) = state.blend {
5027 if bs.color.uses_constant() | bs.alpha.uses_constant() {
5028 flags |= pipeline::PipelineFlags::BLEND_CONSTANT;
5029 }
5030 }
5031 }
5032 if let Some(ds) = depth_stencil_state.as_ref() {
5033 if ds.stencil.is_enabled() && ds.stencil.needs_ref_value() {
5034 flags |= pipeline::PipelineFlags::STENCIL_REFERENCE;
5035 }
5036 if !ds.is_depth_read_only() {
5037 flags |= pipeline::PipelineFlags::WRITES_DEPTH;
5038 }
5039 if !ds.is_stencil_read_only(desc.primitive.cull_mode) {
5040 flags |= pipeline::PipelineFlags::WRITES_STENCIL;
5041 }
5042 }
5043 let shader_modules = {
5044 let mut shader_modules = ArrayVec::new();
5045 match desc.vertex {
5046 pipeline::RenderPipelineVertexProcessor::Vertex(vertex) => {
5047 shader_modules.push(vertex.stage.module)
5048 }
5049 pipeline::RenderPipelineVertexProcessor::Mesh(task, mesh) => {
5050 if let Some(task) = task {
5051 shader_modules.push(task.stage.module);
5052 }
5053 shader_modules.push(mesh.stage.module);
5054 }
5055 }
5056 shader_modules.extend(desc.fragment.map(|f| f.stage.module));
5057 shader_modules
5058 };
5059
5060 let pipeline = pipeline::RenderPipeline {
5061 state: ResourceState::Valid(pipeline::RenderPipelineState {
5062 raw: ManuallyDrop::new(raw),
5063 layout: pipeline_layout.clone(),
5064 }),
5065 device: self.clone(),
5066 pass_context,
5067 _shader_modules: shader_modules,
5068 flags,
5069 topology: desc.primitive.topology,
5070 strip_index_format: desc.primitive.strip_index_format,
5071 vertex_steps,
5072 late_sized_buffer_groups,
5073 immediate_slots_required,
5074 label: desc.label.to_string(),
5075 tracking_data: TrackingData::new(self.tracker_indices.render_pipelines.clone()),
5076 is_mesh,
5077 has_task_shader,
5078 };
5079
5080 let pipeline = Arc::new(pipeline);
5081
5082 if is_auto_layout {
5083 for bgl in pipeline_layout.bind_group_layouts.iter() {
5084 let Some(bgl) = bgl else {
5085 continue;
5086 };
5087
5088 let _ = bgl.exclusive_pipeline.set((&pipeline).into());
5091 }
5092 }
5093
5094 Ok(pipeline)
5095 }
5096
5097 pub unsafe fn create_pipeline_cache(
5101 self: &Arc<Self>,
5102 desc: &pipeline::PipelineCacheDescriptor,
5103 ) -> Result<Arc<pipeline::PipelineCache>, pipeline::CreatePipelineCacheError> {
5104 use crate::pipeline_cache;
5105
5106 self.check_is_valid()?;
5107
5108 self.require_features(wgt::Features::PIPELINE_CACHE)?;
5109 let data = if let Some((data, validation_key)) = desc
5110 .data
5111 .as_ref()
5112 .zip(self.raw().pipeline_cache_validation_key())
5113 {
5114 let data = pipeline_cache::validate_pipeline_cache(
5115 data,
5116 &self.adapter.raw.info,
5117 validation_key,
5118 );
5119 match data {
5120 Ok(data) => Some(data),
5121 Err(e) if e.was_avoidable() || !desc.fallback => return Err(e.into()),
5122 Err(_) => None,
5124 }
5125 } else {
5126 None
5127 };
5128 let cache_desc = hal::PipelineCacheDescriptor {
5129 data,
5130 label: desc.label.to_hal(self.instance_flags),
5131 };
5132 let raw = match unsafe { self.raw().create_pipeline_cache(&cache_desc) } {
5133 Ok(raw) => raw,
5134 Err(e) => match e {
5135 hal::PipelineCacheError::Device(e) => return Err(self.handle_hal_error(e).into()),
5136 },
5137 };
5138 let cache = pipeline::PipelineCache {
5139 device: self.clone(),
5140 label: desc.label.to_string(),
5141 raw: ManuallyDrop::new(raw),
5143 };
5144
5145 let cache = Arc::new(cache);
5146
5147 Ok(cache)
5148 }
5149
5150 fn get_texture_format_features(&self, format: TextureFormat) -> wgt::TextureFormatFeatures {
5151 use wgt::TextureFormatFeatureFlags as tfsc;
5153 let mut format_features = self.adapter.get_texture_format_features(format);
5154 if (format == TextureFormat::R32Float
5155 || format == TextureFormat::Rg32Float
5156 || format == TextureFormat::Rgba32Float)
5157 && !self.features.contains(wgt::Features::FLOAT32_FILTERABLE)
5158 {
5159 format_features.flags.set(tfsc::FILTERABLE, false);
5160 }
5161 format_features
5162 }
5163
5164 pub(crate) fn describe_format_features(
5165 &self,
5166 format: TextureFormat,
5167 ) -> Result<wgt::TextureFormatFeatures, MissingFeatures> {
5168 self.require_features(format.required_features())?;
5169
5170 let using_device_features = self
5171 .features
5172 .contains(wgt::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES);
5173 let downlevel = !self
5176 .downlevel
5177 .flags
5178 .contains(wgt::DownlevelFlags::WEBGPU_TEXTURE_FORMAT_SUPPORT);
5179
5180 if using_device_features || downlevel {
5181 Ok(self.get_texture_format_features(format))
5182 } else {
5183 Ok(format.guaranteed_format_features(self.features))
5184 }
5185 }
5186
5187 #[cfg(feature = "replay")]
5188 pub(crate) fn wait_for_submit(
5189 &self,
5190 submission_index: crate::SubmissionIndex,
5191 ) -> Result<(), DeviceError> {
5192 let last_done_index = unsafe { self.raw().get_fence_value(self.fence.as_ref()) }
5193 .map_err(|e| self.handle_hal_error(e))?;
5194 if last_done_index < submission_index {
5195 unsafe { self.raw().wait(self.fence.as_ref(), submission_index, None) }
5196 .map_err(|e| self.handle_hal_error(e))?;
5197 if let Some(queue) = self.get_queue() {
5198 let closures = queue.lock_life().triage_submissions(submission_index);
5199 assert!(
5200 closures.is_empty(),
5201 "wait_for_submit is not expected to work with closures"
5202 );
5203 }
5204 }
5205 Ok(())
5206 }
5207
5208 pub fn create_query_set(
5209 self: &Arc<Self>,
5210 desc: &resource::QuerySetDescriptor,
5211 ) -> Result<Arc<QuerySet>, resource::CreateQuerySetError> {
5212 use resource::CreateQuerySetError as Error;
5213
5214 self.check_is_valid()?;
5215
5216 match desc.ty {
5217 wgt::QueryType::Occlusion => {}
5218 wgt::QueryType::Timestamp => {
5219 self.require_features(wgt::Features::TIMESTAMP_QUERY)?;
5220 }
5221 wgt::QueryType::PipelineStatistics(..) => {
5222 self.require_features(wgt::Features::PIPELINE_STATISTICS_QUERY)?;
5223 }
5224 }
5225
5226 if desc.count == 0 {
5227 return Err(Error::ZeroCount);
5228 }
5229
5230 if desc.count > wgt::QUERY_SET_MAX_QUERIES {
5231 return Err(Error::TooManyQueries {
5232 count: desc.count,
5233 maximum: wgt::QUERY_SET_MAX_QUERIES,
5234 });
5235 }
5236
5237 let hal_desc = desc.map_label(|label| label.to_hal(self.instance_flags));
5238
5239 let raw = unsafe { self.raw().create_query_set(&hal_desc) }
5240 .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
5241
5242 let query_set = QuerySet {
5243 raw: Snatchable::new(raw),
5244 device: self.clone(),
5245 label: desc.label.to_string(),
5246 tracking_data: TrackingData::new(self.tracker_indices.query_sets.clone()),
5247 desc: desc.map_label(|_| ()),
5248 initialized_slots: Mutex::new(
5249 rank::QUERY_SET_INITIALIZED_SLOTS,
5250 bit_vec::BitVec::from_elem(desc.count as usize, false),
5251 ),
5252 };
5253
5254 let query_set = Arc::new(query_set);
5255
5256 Ok(query_set)
5257 }
5258
5259 pub fn configure_surface(
5260 self: &Arc<Self>,
5261 surface: &crate::instance::Surface,
5262 config: &wgt::SurfaceConfiguration<Vec<TextureFormat>>,
5263 ) -> Option<present::ConfigureSurfaceError> {
5264 use present::ConfigureSurfaceError as E;
5265 profiling::scope!("surface_configure");
5266
5267 log::debug!("configuring surface with {config:?}");
5268
5269 let error = 'error: {
5270 let user_callbacks;
5272 {
5273 if let Err(e) = self.check_is_valid() {
5274 break 'error e.into();
5275 }
5276
5277 let caps = match surface.get_capabilities(&self.adapter) {
5278 Ok(caps) => caps,
5279 Err(_) => break 'error E::UnsupportedQueueFamily,
5280 };
5281
5282 let mut hal_view_formats = Vec::new();
5283 for format in config.view_formats.iter() {
5284 if *format == config.format {
5285 continue;
5286 }
5287 if !caps.formats.iter().any(|fc| fc.format == config.format) {
5288 break 'error E::UnsupportedFormat {
5289 requested: config.format,
5290 available: caps.texture_formats().collect(),
5291 };
5292 }
5293 if config.format.remove_srgb_suffix() != format.remove_srgb_suffix() {
5294 break 'error E::InvalidViewFormat(*format, config.format);
5295 }
5296 hal_view_formats.push(*format);
5297 }
5298
5299 if !hal_view_formats.is_empty() {
5300 if let Err(missing_flag) =
5301 self.require_downlevel_flags(wgt::DownlevelFlags::SURFACE_VIEW_FORMATS)
5302 {
5303 break 'error E::MissingDownlevelFlags(missing_flag);
5304 }
5305 }
5306
5307 let maximum_frame_latency = config.desired_maximum_frame_latency.clamp(
5308 *caps.maximum_frame_latency.start(),
5309 *caps.maximum_frame_latency.end(),
5310 );
5311 let mut hal_config = hal::SurfaceConfiguration {
5312 maximum_frame_latency,
5313 present_mode: config.present_mode,
5314 composite_alpha_mode: config.alpha_mode,
5315 format: config.format,
5316 color_space: config.color_space,
5317 extent: wgt::Extent3d {
5318 width: config.width,
5319 height: config.height,
5320 depth_or_array_layers: 1,
5321 },
5322 usage: conv::map_texture_usage(
5323 config.usage,
5324 hal::FormatAspects::COLOR,
5325 wgt::TextureFormatFeatureFlags::STORAGE_READ_ONLY
5326 | wgt::TextureFormatFeatureFlags::STORAGE_WRITE_ONLY
5327 | wgt::TextureFormatFeatureFlags::STORAGE_READ_WRITE,
5328 ),
5329 view_formats: hal_view_formats,
5330 };
5331
5332 if let Err(error) = validate_surface_configuration(
5333 &mut hal_config,
5334 &caps,
5335 self.limits.max_texture_dimension_2d,
5336 ) {
5337 break 'error error;
5338 }
5339
5340 let snatch_guard = self.snatchable_lock.read();
5342
5343 let maintain_result;
5344 (user_callbacks, maintain_result) =
5345 self.maintain(wgt::PollType::wait_indefinitely(), snatch_guard);
5346
5347 match maintain_result {
5348 Ok(wgt::PollStatus::QueueEmpty) => {}
5350 Ok(wgt::PollStatus::WaitSucceeded) => {
5351 break 'error E::GpuWaitTimeout;
5354 }
5355 Ok(wgt::PollStatus::Poll) => {
5356 unreachable!("Cannot get a Poll result from a Wait action.")
5357 }
5358 Err(WaitIdleError::Timeout) if cfg!(target_arch = "wasm32") => {
5359 }
5364 Err(e) => {
5365 break 'error e.into();
5366 }
5367 }
5368
5369 if let Some(present) = surface.presentation.lock().take() {
5371 if present.acquired_texture.is_some() {
5372 break 'error E::PreviousOutputExists;
5373 }
5374 }
5375
5376 let surface_raw = surface.raw(self.backend()).unwrap();
5383 match unsafe { surface_raw.configure(self.raw(), &hal_config) } {
5384 Ok(()) => (),
5385 Err(error) => {
5386 break 'error match error {
5387 hal::SurfaceError::Outdated
5388 | hal::SurfaceError::Lost
5389 | hal::SurfaceError::Occluded
5390 | hal::SurfaceError::Timeout => E::InvalidSurface,
5391 hal::SurfaceError::Device(error) => {
5392 E::Device(self.handle_hal_error(error))
5393 }
5394 hal::SurfaceError::Other(message) => {
5395 log::error!("surface configuration failed: {message}");
5396 E::InvalidSurface
5397 }
5398 }
5399 }
5400 }
5401
5402 let mut presentation = surface.presentation.lock();
5403 *presentation = Some(present::Presentation {
5404 device: Arc::clone(self),
5405 config: config.clone(),
5406 acquired_texture: None,
5407 });
5408 }
5409
5410 user_callbacks.fire();
5411 return None;
5412 };
5413
5414 Some(error)
5415 }
5416
5417 fn lose(&self, message: &str) {
5418 self.valid.store(false, Ordering::Release);
5423
5424 if let Some(device_lost_closure) = self.device_lost_closure.lock().take() {
5426 device_lost_closure(DeviceLostReason::Unknown, message.to_string());
5427 }
5428
5429 }
5437
5438 fn release_gpu_resources(&self) {
5439 let trackers = self.trackers.lock();
5449 for buffer in trackers.buffers.used_resources() {
5450 if let Some(buffer) = Weak::upgrade(buffer) {
5451 buffer.destroy();
5452 }
5453 }
5454 for texture in trackers.textures.used_resources() {
5455 if let Some(texture) = Weak::upgrade(texture) {
5456 texture.destroy();
5457 }
5458 }
5459 }
5460
5461 pub(crate) fn new_usage_scope(&self) -> UsageScope<'_> {
5462 UsageScope::new_pooled(
5463 &self.usage_scopes,
5464 &self.tracker_indices,
5465 self.ordered_buffer_usages,
5466 self.ordered_texture_usages,
5467 )
5468 }
5469
5470 pub fn get_hal_counters(&self) -> wgt::HalCounters {
5471 self.raw().get_internal_counters()
5472 }
5473
5474 pub fn generate_allocator_report(&self) -> Option<wgt::AllocatorReport> {
5475 self.raw().generate_allocator_report()
5476 }
5477}
5478
5479crate::impl_resource_type!(Device);
5480crate::impl_labeled!(Device);
5481crate::impl_storage_item!(Device);