#![deny(unsafe_op_in_unsafe_fn)] use core::marker::PhantomData; use std::alloc; use std::cell::Cell; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; trait PtrCellTrait: std::fmt::Debug { fn new(val: *mut u8) -> Self; fn get(&self) -> *mut u8; /// Modifies the wrapped value. fn set(&mut self, val: *mut u8); /// Performs an atomic compare-and-swap on the wrapped value. /// /// If the compare succeeded and `cb` returns `Some(new_ptr)`, stored `new_ptr` and returns `Ok(old_ptr). /// /// If `cb` returns `None`, returns `Err(old_ptr)`. /// /// May retry multiple times if the comparison fails. fn try_update(&self, cb: impl Fn(*mut u8) -> Option<*mut u8>) -> Result<*mut u8, *mut u8>; } impl PtrCellTrait for Cell<*mut u8> { fn new(val: *mut u8) -> Self { Cell::new(val) } #[inline(always)] fn get(&self) -> *mut u8 { Cell::get(self) } #[inline(always)] fn set(&mut self, val: *mut u8) { Cell::set(self, val) } #[inline(always)] fn try_update(&self, cb: impl Fn(*mut u8) -> Option<*mut u8>) -> Result<*mut u8, *mut u8> { let val = Cell::get(self); if let Some(new_val) = cb(val) { Cell::set(self, new_val); Ok(val) } else { Err(val) } } } impl PtrCellTrait for AtomicPtr { fn new(val: *mut u8) -> Self { AtomicPtr::new(val) } #[inline(always)] fn get(&self) -> *mut u8 { self.load(Ordering::Acquire) } #[inline(always)] fn set(&mut self, val: *mut u8) { *self.get_mut() = val; } #[inline] fn try_update(&self, cb: impl Fn(*mut u8) -> Option<*mut u8>) -> Result<*mut u8, *mut u8> { let mut prev = PtrCellTrait::get(self); while let Some(next) = cb(prev) { match self.compare_exchange_weak(prev, next, Ordering::Relaxed, Ordering::Acquire) { x @ Ok(_) => return x, Err(next_prev) => prev = next_prev, } } Err(prev) // TODO: replace with the following once 1.95 is the msrv: // AtomicPtr::try_update(self, Ordering::Relaxed, Ordering::Acquire, cb) } } /// Allows the choice of implementation for the mutable pointer in [`RawBlock`]. /// Can be one of: /// - [`RawBlockSerial`] (using a [`Cell<*mut u8>`]) /// - [`RawBlockConcurrent`] (using a [`AtomicPtr`]) pub(crate) trait RawBlockConcurrency { #[allow(private_bounds)] type PtrCell: PtrCellTrait; } #[derive(Debug, Clone, Copy)] pub struct RawBlockSerial(); #[derive(Debug, Clone, Copy)] pub struct RawBlockConcurrent(); impl RawBlockConcurrency for RawBlockSerial { type PtrCell = Cell<*mut u8>; } impl RawBlockConcurrency for RawBlockConcurrent { type PtrCell = AtomicPtr; } use crate::machine::heap::AllocError; pub trait RawBlockTraits { /// ## Safety /// /// Must be non-zero. fn init_size() -> usize; /// ## Safety /// /// Must be constant. /// /// Must respect the invariants of [`std::alloc::Layout::from_size_align()`], namely: /// - must not be zero /// - must be a power of two /// - must not overflow `usize` fn align() -> usize; } /// A block of memory with fast, lock-free appends. /// /// ## Invariants /// /// - `base.is_null()` iff `capacity == 0` /// - if `!base.is_null()`, then `ptr.get()` is in the same allocation as `base`. #[derive(Debug)] pub struct RawBlock { base: *const u8, capacity: usize, ptr: C::PtrCell, _marker: PhantomData, _c_marker: PhantomData, } impl RawBlock { #[inline] pub fn empty_block() -> Self { RawBlock { base: ptr::null(), capacity: 0, ptr: C::PtrCell::new(ptr::null_mut()), _marker: PhantomData, _c_marker: PhantomData, } } #[allow(clippy::new_without_default)] pub fn new() -> Result { let mut block = Self::empty_block(); unsafe { block.grow()?; } Ok(block) } /// ## Safety /// /// Assumes that the object has not been initialized before (ie. `self.base.is_null()`) /// and assumes that `cap > 0`. unsafe fn init_at_size(&mut self, cap: usize) -> Result<(), AllocError> { debug_assert!(cap > 0); debug_assert!(self.base.is_null()); // SAFETY: // - Guaranteed by caller: `cap > 0` // - Guaranteed by caller: `T::align()` respects the invariants of `Layout::from_size_align` let new_base = unsafe { let layout = alloc::Layout::from_size_align_unchecked(cap, T::align()); alloc::alloc(layout).cast_const() }; if new_base.is_null() { return Err(AllocError); } self.base = new_base; self.capacity = cap; self.ptr.set(self.base.cast_mut()); Ok(()) } /// ## Safety /// /// Invalidates all pointers previously obtained by [`RawBlock::get()`] or [`RawBlock::alloc()`]. pub unsafe fn grow(&mut self) -> Result<(), AllocError> { self.debug_check_invariants(); if self.base.is_null() { // SAFETY: // - Guaranteed by caller: `T::init_size() > 0` // - Asserted: `self.base.is_null()` unsafe { self.init_at_size(T::init_size()) } } else { let size = self.capacity(); let used_bytes = self.used_bytes(); // SAFETY: // - Guaranteed by caller: `T::align()` respects the invariants of `Layout::from_size_align` // - Asserted: `!self.base.is_null()` // - Invariant: `self.base.is_null()` iff `self.capacity() == 0` // - Thus `self.capacity() > 0` let new_base = unsafe { let layout = alloc::Layout::from_size_align_unchecked(size, T::align()); alloc::realloc(self.base.cast_mut(), layout, size * 2).cast_const() }; if new_base.is_null() { Err(AllocError) } else { self.base = new_base; self.capacity = size * 2; // SAFETY: // - Invariant: `used_bytes < size` // - Definition: `new_base` has allocation size `2 * size` let new_ptr = unsafe { self.base.add(used_bytes).cast_mut() }; self.ptr.set(new_ptr); self.debug_check_invariants(); Ok(()) } } } pub unsafe fn grow_new(&self) -> Result { self.debug_check_invariants(); if self.base.is_null() { Self::new() } else { let mut new_block = Self::empty_block(); // SAFETY: // - Asserted: !self.base.is_null() // - Invariant: self.base.is_null() iff self.capacity == 0 // - Thus self.capacity > 0 // - Definition: `new_block` was not yet initialized unsafe { new_block.init_at_size(self.capacity() * 2)?; } let used_bytes = self.used_bytes(); // SAFETY: // - Definition: `self.base` contains `self.allocated()` bytes // - Invariant: `self.used_bytes() < self.allocated()` unsafe { self.base.copy_to(new_block.base.cast_mut(), used_bytes); new_block.ptr.set(new_block.base.add(used_bytes).cast_mut()); } new_block.debug_check_invariants(); Ok(new_block) } } #[inline(always)] fn debug_check_invariants(&self) { if cfg!(debug_assertions) { assert!( self.ptr.get().cast_const() >= self.base, "self.ptr = {:?} < {:?} = self.base", self.ptr.get(), self.base ); assert!(self.used_bytes() <= self.capacity()); } } #[inline] pub fn capacity(&self) -> usize { self.capacity } #[inline] pub fn used_bytes(&self) -> usize { // SAFETY: // - Invariant: `ptr` is in the same allocation as `base` unsafe { self.ptr.get().offset_from(self.base) as usize } } pub unsafe fn alloc(&self, size: usize) -> *mut u8 { self.debug_check_invariants(); let aligned_size = size.next_multiple_of(T::align()); match self.ptr.try_update(|ptr| { // SAFETY: // - Invariant: `ptr` is in the same allocation as `base` let free_bytes = unsafe { self.capacity() - ptr.offset_from(self.base) as usize }; if free_bytes >= aligned_size { Some(unsafe { ptr.add(aligned_size) }) } else { // Not enough space: don't allocate and return a null pointer None } }) { Ok(ptr) => ptr, Err(_) => ptr::null_mut(), } } /// Moves `ptr` back to `new_size`. /// /// Note that this method does *not* deallocate what was placed in the [`RawBlock`]. /// Pointers to data past `new_size` remain valid until the next call to [`RawBlock::alloc()`]. pub fn shift_back(&mut self, new_size: usize) { self.debug_check_invariants(); assert!( new_size <= self.used_bytes(), "Shrink cannot grow: new_size = {:?} > allocated = {:?}", new_size, self.used_bytes() ); // SAFETY: // - Asserted: new_size <= self.capacity // - Definition: self.base := alloc(self.capacity) let new_ptr = unsafe { self.base.add(new_size) }; debug_assert!(new_ptr as usize <= self.ptr.get() as usize,); self.ptr.set(new_ptr.cast_mut()); self.debug_check_invariants(); } /// Returns a pointer at a given `offset` within the block of memory. /// /// Panics if that range of bytes wasn't allocated yet with [`RawBlock::alloc()`]. pub fn get(&self, offset: usize) -> *const u8 { assert!(offset < self.used_bytes()); // SAFETY: Asserted. unsafe { self.get_unchecked(offset) } } /// Returns a pointer at a given `offset` within the block of memory. /// /// ## Safety /// /// Assumes that `offset < self.capacity()`. #[inline] pub unsafe fn get_unchecked(&self, offset: usize) -> *const u8 { debug_assert!( offset < self.capacity(), "offset out of bounds: offset is {:?} but {:?} bytes are available", offset, self.used_bytes() ); // SAFETY: Guaranteed by caller. unsafe { self.base.add(offset) } } /// ## Safety /// /// `ptr` is a valid pointer be obtained from [`RawBlock::get()`] or [`RawBlock::alloc()`]. #[inline] pub unsafe fn get_offset(&self, ptr: *const u8) -> usize { // SAFETY: // - Guaranteed by caller: `ptr` is still valid // - Guranteed by caller: `ptr` was obtained from `get()` or `alloc()` // - get() and alloc() return pointers in the same allocation as `self.base` // - All functions modifying `self.base` invalidate pointers in their contract // - Thus `ptr` and `self.base` originate from the same allocation unsafe { ptr.offset_from(self.base) as usize } } } impl Drop for RawBlock { fn drop(&mut self) { if !self.base.is_null() { unsafe { let layout = alloc::Layout::from_size_align_unchecked(self.capacity(), T::align()); alloc::dealloc(self.base as *mut _, layout); } } } } impl From> for RawBlock { fn from(other: RawBlock) -> Self { Self { base: other.base, capacity: other.capacity, ptr: PtrCellTrait::new(other.ptr.get()), _marker: PhantomData, _c_marker: PhantomData, } } } impl From> for RawBlock { fn from(other: RawBlock) -> Self { Self { base: other.base, capacity: other.capacity, ptr: PtrCellTrait::new(other.ptr.get()), _marker: PhantomData, _c_marker: PhantomData, } } }