1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
use {
crate::{align_down, align_up, error::MapError},
alloc::sync::Arc,
core::{
convert::TryFrom as _,
ptr::{copy_nonoverlapping, NonNull},
// sync::atomic::{AtomicU8, Ordering::*},
},
gpu_alloc_types::{MappedMemoryRange, MemoryDevice, MemoryPropertyFlags},
};
#[derive(Debug)]
struct Relevant;
impl Drop for Relevant {
fn drop(&mut self) {
report_error_on_drop!("Memory block wasn't deallocated");
}
}
/// Memory block allocated by `GpuAllocator`.
#[derive(Debug)]
pub struct MemoryBlock<M> {
memory_type: u32,
props: MemoryPropertyFlags,
offset: u64,
size: u64,
atom_mask: u64,
mapped: bool,
flavor: MemoryBlockFlavor<M>,
relevant: Relevant,
}
impl<M> MemoryBlock<M> {
pub(crate) fn new(
memory_type: u32,
props: MemoryPropertyFlags,
offset: u64,
size: u64,
atom_mask: u64,
flavor: MemoryBlockFlavor<M>,
) -> Self {
isize::try_from(atom_mask).expect("`atom_mask` is too large");
MemoryBlock {
memory_type,
props,
offset,
size,
atom_mask,
flavor,
mapped: false,
relevant: Relevant,
}
}
pub(crate) fn deallocate(self) -> MemoryBlockFlavor<M> {
core::mem::forget(self.relevant);
self.flavor
}
}
unsafe impl<M> Sync for MemoryBlock<M> where M: Sync {}
unsafe impl<M> Send for MemoryBlock<M> where M: Send {}
#[derive(Debug)]
pub(crate) enum MemoryBlockFlavor<M> {
Dedicated {
memory: M,
},
Buddy {
chunk: usize,
index: usize,
ptr: Option<NonNull<u8>>,
memory: Arc<M>,
},
FreeList {
chunk: u64,
ptr: Option<NonNull<u8>>,
memory: Arc<M>,
},
}
impl<M> MemoryBlock<M> {
/// Returns reference to parent memory object.
#[inline(always)]
pub fn memory(&self) -> &M {
match &self.flavor {
MemoryBlockFlavor::Dedicated { memory } => memory,
MemoryBlockFlavor::Buddy { memory, .. } => memory,
MemoryBlockFlavor::FreeList { memory, .. } => memory,
}
}
/// Returns offset in bytes from start of memory object to start of this block.
#[inline(always)]
pub fn offset(&self) -> u64 {
self.offset
}
/// Returns size of this memory block.
#[inline(always)]
pub fn size(&self) -> u64 {
self.size
}
/// Returns memory property flags for parent memory object.
#[inline(always)]
pub fn props(&self) -> MemoryPropertyFlags {
self.props
}
/// Returns index of type of parent memory object.
#[inline(always)]
pub fn memory_type(&self) -> u32 {
self.memory_type
}
/// Returns pointer to mapped memory range of this block.
/// This blocks becomes mapped.
///
/// The user of returned pointer must guarantee that any previously submitted command that writes to this range has completed
/// before the host reads from or writes to that range,
/// and that any previously submitted command that reads from that range has completed
/// before the host writes to that region.
/// If the device memory was allocated without the `HOST_COHERENT` property flag set,
/// these guarantees must be made for an extended range:
/// the user must round down the start of the range to the nearest multiple of `non_coherent_atom_size`,
/// and round the end of the range up to the nearest multiple of `non_coherent_atom_size`.
///
/// # Panics
///
/// This function panics if block is currently mapped.
///
/// # Safety
///
/// `block` must have been allocated from specified `device`.
#[inline(always)]
pub unsafe fn map(
&mut self,
device: &impl MemoryDevice<M>,
offset: u64,
size: usize,
) -> Result<NonNull<u8>, MapError> {
let size_u64 = u64::try_from(size).expect("`size` doesn't fit device address space");
assert!(offset < self.size, "`offset` is out of memory block bounds");
assert!(
size_u64 <= self.size - offset,
"`offset + size` is out of memory block bounds"
);
let ptr = match &mut self.flavor {
MemoryBlockFlavor::Dedicated { memory } => {
let end = align_up(offset + size_u64, self.atom_mask)
.expect("mapping end doesn't fit device address space");
let aligned_offset = align_down(offset, self.atom_mask);
if !acquire_mapping(&mut self.mapped) {
return Err(MapError::AlreadyMapped);
}
let result =
device.map_memory(memory, self.offset + aligned_offset, end - aligned_offset);
match result {
// the overflow is checked in `Self::new()`
Ok(ptr) => {
let ptr_offset = (offset - aligned_offset) as isize;
ptr.as_ptr().offset(ptr_offset)
}
Err(err) => {
release_mapping(&mut self.mapped);
return Err(err.into());
}
}
}
MemoryBlockFlavor::FreeList { ptr: Some(ptr), .. }
| MemoryBlockFlavor::Buddy { ptr: Some(ptr), .. } => {
if !acquire_mapping(&mut self.mapped) {
return Err(MapError::AlreadyMapped);
}
let offset_isize = isize::try_from(offset)
.expect("Buddy and linear block should fit host address space");
ptr.as_ptr().offset(offset_isize)
}
_ => return Err(MapError::NonHostVisible),
};
Ok(NonNull::new_unchecked(ptr))
}
/// Unmaps memory range of this block that was previously mapped with `Block::map`.
/// This block becomes unmapped.
///
/// # Panics
///
/// This function panics if this block is not currently mapped.
///
/// # Safety
///
/// `block` must have been allocated from specified `device`.
#[inline(always)]
pub unsafe fn unmap(&mut self, device: &impl MemoryDevice<M>) -> bool {
if !release_mapping(&mut self.mapped) {
return false;
}
match &mut self.flavor {
MemoryBlockFlavor::Dedicated { memory } => {
device.unmap_memory(memory);
}
MemoryBlockFlavor::Buddy { .. } => {}
MemoryBlockFlavor::FreeList { .. } => {}
}
true
}
/// Transiently maps block memory range and copies specified data
/// to the mapped memory range.
///
/// # Panics
///
/// This function panics if block is currently mapped.
///
/// # Safety
///
/// `block` must have been allocated from specified `device`.
/// The caller must guarantee that any previously submitted command that reads or writes to this range has completed.
#[inline(always)]
pub unsafe fn write_bytes(
&mut self,
device: &impl MemoryDevice<M>,
offset: u64,
data: &[u8],
) -> Result<(), MapError> {
let size = data.len();
let ptr = self.map(device, offset, size)?;
copy_nonoverlapping(data.as_ptr(), ptr.as_ptr(), size);
let result = if !self.coherent() {
let aligned_offset = align_down(offset, self.atom_mask);
let end = align_up(offset + data.len() as u64, self.atom_mask).unwrap();
device.flush_memory_ranges(&[MappedMemoryRange {
memory: self.memory(),
offset: self.offset + aligned_offset,
size: end - aligned_offset,
}])
} else {
Ok(())
};
self.unmap(device);
result.map_err(Into::into)
}
/// Transiently maps block memory range and copies specified data
/// from the mapped memory range.
///
/// # Panics
///
/// This function panics if block is currently mapped.
///
/// # Safety
///
/// `block` must have been allocated from specified `device`.
/// The caller must guarantee that any previously submitted command that reads to this range has completed.
#[inline(always)]
pub unsafe fn read_bytes(
&mut self,
device: &impl MemoryDevice<M>,
offset: u64,
data: &mut [u8],
) -> Result<(), MapError> {
#[cfg(feature = "tracing")]
{
if !self.cached() {
tracing::warn!("Reading from non-cached memory may be slow. Consider allocating HOST_CACHED memory block for host reads.")
}
}
let size = data.len();
let ptr = self.map(device, offset, size)?;
let result = if !self.coherent() {
let aligned_offset = align_down(offset, self.atom_mask);
let end = align_up(offset + data.len() as u64, self.atom_mask).unwrap();
device.invalidate_memory_ranges(&[MappedMemoryRange {
memory: self.memory(),
offset: self.offset + aligned_offset,
size: end - aligned_offset,
}])
} else {
Ok(())
};
if result.is_ok() {
copy_nonoverlapping(ptr.as_ptr(), data.as_mut_ptr(), size);
}
self.unmap(device);
result.map_err(Into::into)
}
fn coherent(&self) -> bool {
self.props.contains(MemoryPropertyFlags::HOST_COHERENT)
}
#[cfg(feature = "tracing")]
fn cached(&self) -> bool {
self.props.contains(MemoryPropertyFlags::HOST_CACHED)
}
}
fn acquire_mapping(mapped: &mut bool) -> bool {
if *mapped {
false
} else {
*mapped = true;
true
}
}
fn release_mapping(mapped: &mut bool) -> bool {
if *mapped {
*mapped = false;
true
} else {
false
}
}