rishi/scryer-prolog
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use OffsetTableImpl without synchronization by default

Browse Source
This commit is contained in:
Mark Thom
2025-05-22 23:49:57 -07:00
committed by Mark Thom
parent 725def07cd
commit 1332611f83
28 changed files with 1005 additions and 990 deletions
Download Patch File Download Diff File Expand all files Collapse all files

541
src/offset_table.rs
View File

@@ -1,8 +1,6 @@
use std::cell::UnsafeCell;
use std::hash::{Hash, Hasher};
use std::ops::{Deref, DerefMut};
use std::sync::RwLock;
use std::sync::Weak;
use std::sync::{Arc, Mutex};
use std::{fmt, mem, ptr};
@@ -23,19 +21,7 @@ const F64_TABLE_ALIGN: usize = 8;
const CODE_INDEX_TABLE_INIT_SIZE: usize = 1 << 16;
const CODE_INDEX_TABLE_ALIGN: usize = 8;
#[derive(Debug)]
pub struct OffsetTableImpl<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
block: Arcu<RawBlock<OffsetTableImpl<T>>, GlobalEpochCounterPool>,
update: Mutex<()>,
}
pub type F64Table = OffsetTableImpl<OrderedFloat<f64>>;
pub type CodeIndexTable = OffsetTableImpl<IndexPtr>;
impl RawBlockTraits for F64Table {
impl RawBlockTraits for OrderedFloat<f64> {
#[inline]
fn init_size() -> usize {
F64_TABLE_INIT_SIZE
@@ -47,7 +33,7 @@ impl RawBlockTraits for F64Table {
}
}
impl RawBlockTraits for CodeIndexTable {
impl RawBlockTraits for IndexPtr {
#[inline]
fn init_size() -> usize {
CODE_INDEX_TABLE_INIT_SIZE
@@ -59,72 +45,159 @@ impl RawBlockTraits for CodeIndexTable {
}
}
pub trait OffsetTable: RawBlockTraits {
type Offset: Copy + From<usize> + Into<usize>;
type Stored;
fn global_table() -> &'static RwLock<Weak<Self>>;
}
impl OffsetTable for F64Table {
type Offset = F64Offset;
type Stored = OrderedFloat<f64>;
#[derive(Debug)]
pub struct OffsetTableImpl<T: RawBlockTraits>(InnerOffsetTableImpl<T>);
impl<T: RawBlockTraits> OffsetTableImpl<T> {
#[inline(always)]
fn global_table() -> &'static RwLock<Weak<Self>> {
static GLOBAL_ATOM_TABLE: RwLock<Weak<F64Table>> = RwLock::new(Weak::new());
&GLOBAL_ATOM_TABLE
pub fn new() -> Self {
Self(InnerOffsetTableImpl::Serial(SerialOffsetTable::new()))
}
}
impl OffsetTable for CodeIndexTable {
type Offset = CodeIndexOffset;
type Stored = IndexPtr;
#[inline(always)]
fn global_table() -> &'static RwLock<Weak<CodeIndexTable>> {
static GLOBAL_CODE_INDEX_TABLE: RwLock<Weak<CodeIndexTable>> = RwLock::new(Weak::new());
&GLOBAL_CODE_INDEX_TABLE
impl<T: RawBlockTraits> Default for OffsetTableImpl<T> {
fn default() -> Self {
Self::new()
}
}
impl<T: 'static> OffsetTableImpl<T>
where
OffsetTableImpl<T>: OffsetTable<Stored = T>,
{
#[inline]
pub fn new() -> Arc<Self> {
let upgraded = Self::global_table().read().unwrap().upgrade();
// don't inline upgraded, otherwise temporary will be dropped too late in case of None
if let Some(atom_table) = upgraded {
atom_table
} else {
let mut guard = Self::global_table().write().unwrap();
// try to upgrade again in case we lost the race on the write lock
if let Some(atom_table) = guard.upgrade() {
atom_table
} else {
let table = Arc::new(Self {
block: Arcu::new(RawBlock::new(), GlobalEpochCounterPool),
update: Mutex::new(()),
});
*guard = Arc::downgrade(&table);
table
}
#[derive(Debug)]
enum InnerOffsetTableImpl<T: RawBlockTraits> {
Serial(SerialOffsetTable<T>),
#[allow(dead_code)]
Concurrent(Arc<ConcurrentOffsetTable<T>>),
}
impl<T: RawBlockTraits> InnerOffsetTableImpl<T> {
#[inline(always)]
fn build_with(&mut self, value: T) -> usize {
match self {
Self::Concurrent(concurrent_tbl) => unsafe { concurrent_tbl.build_with(value) },
Self::Serial(serial_tbl) => unsafe { serial_tbl.build_with(value) },
}
}
#[inline(always)]
fn lookup<'a>(&'a self, offset: usize) -> TablePtr<'a, T> {
match self {
Self::Concurrent(concurrent_tbl) => {
TablePtr(InnerTablePtr::Concurrent(concurrent_tbl.lookup(offset)))
}
Self::Serial(serial_tbl) => unsafe {
TablePtr(InnerTablePtr::Serial(serial_tbl.lookup(offset)))
},
}
}
#[inline(always)]
fn lookup_mut<'a>(&'a mut self, offset: usize) -> TablePtrMut<'a, T> {
match self {
Self::Concurrent(concurrent_tbl) => TablePtrMut(InnerTablePtrMut::Concurrent(
concurrent_tbl.lookup_mut(offset),
)),
Self::Serial(serial_tbl) => unsafe {
TablePtrMut(InnerTablePtrMut::Serial(serial_tbl.lookup_mut(offset)))
},
}
}
}
pub trait OffsetTable<T: RawBlockTraits> {
type Offset: Copy + Into<usize>;
fn build_with(&mut self, value: T) -> Self::Offset;
fn lookup<'a>(&'a self, offset: Self::Offset) -> TablePtr<'a, T>;
fn lookup_mut<'a>(&'a mut self, offset: Self::Offset) -> TablePtrMut<'a, T>;
}
impl OffsetTable<OrderedFloat<f64>> for OffsetTableImpl<OrderedFloat<f64>> {
type Offset = F64Offset;
fn build_with(&mut self, value: OrderedFloat<f64>) -> F64Offset {
F64Offset(self.0.build_with(value))
}
fn lookup<'a>(&'a self, offset: F64Offset) -> TablePtr<'a, OrderedFloat<f64>> {
self.0.lookup(offset.into())
}
fn lookup_mut<'a>(&'a mut self, offset: F64Offset) -> TablePtrMut<'a, OrderedFloat<f64>> {
self.0.lookup_mut(offset.into())
}
}
impl OffsetTable<IndexPtr> for OffsetTableImpl<IndexPtr> {
type Offset = CodeIndexOffset;
fn build_with(&mut self, value: IndexPtr) -> CodeIndexOffset {
CodeIndexOffset(self.0.build_with(value))
}
fn lookup<'a>(&'a self, offset: CodeIndexOffset) -> TablePtr<'a, IndexPtr> {
self.0.lookup(offset.into())
}
fn lookup_mut<'a>(&'a mut self, offset: CodeIndexOffset) -> TablePtrMut<'a, IndexPtr> {
self.0.lookup_mut(offset.into())
}
}
#[derive(Debug)]
struct SerialOffsetTable<T: RawBlockTraits> {
block: RawBlock<T>,
}
impl<T: RawBlockTraits> SerialOffsetTable<T> {
#[inline]
fn new() -> Self {
Self {
block: RawBlock::new(),
}
}
unsafe fn build_with(&mut self, value: T) -> usize {
let mut ptr;
loop {
ptr = self.block.alloc(size_of::<T>());
if ptr.is_null() {
let new_block = self.block.grow_new().unwrap();
self.block = new_block;
} else {
break;
}
}
ptr::write(ptr as *mut T, value);
ptr.addr() - self.block.base.addr()
}
#[inline]
unsafe fn lookup(&self, offset: usize) -> &T {
&*self.block.base.add(offset).cast::<T>()
}
#[inline]
unsafe fn lookup_mut(&mut self, offset: usize) -> &mut T {
&mut *self.block.base.add(offset).cast::<T>().cast_mut()
}
}
#[derive(Debug)]
pub struct ConcurrentOffsetTable<T: RawBlockTraits> {
block: Arcu<RawBlock<T>, GlobalEpochCounterPool>,
update: Mutex<()>,
}
impl<T: RawBlockTraits> ConcurrentOffsetTable<T> {
#[allow(clippy::missing_safety_doc)]
pub unsafe fn build_with(
&self,
value: <OffsetTableImpl<T> as OffsetTable>::Stored,
) -> <OffsetTableImpl<T> as OffsetTable>::Offset {
unsafe fn build_with(&self, value: T) -> usize {
let update_guard = self.update.lock();
// we don't have an index table for lookups as AtomTable does so
// just get the epoch after we take the upgrade lock
let mut block_epoch = self.block.read();
let mut ptr;
loop {
@@ -141,8 +214,7 @@ where
ptr::write(ptr as *mut T, value);
let value =
<OffsetTableImpl<T> as OffsetTable>::Offset::from(ptr.addr() - block_epoch.base.addr());
let value = ptr.addr() - block_epoch.base.addr();
// AtomTable would have to update the index table at this point
// explicit drop to ensure we don't accidentally drop it early
@@ -151,17 +223,20 @@ where
value
}
pub fn lookup(offset: <Self as OffsetTable>::Offset) -> RcuRef<RawBlock<Self>, UnsafeCell<T>> {
let table = Self::global_table()
.read()
.unwrap()
.upgrade()
.expect("We should only be looking up entries when there is a table");
#[inline]
fn lookup(&self, offset: usize) -> RcuRef<RawBlock<T>, T> {
RcuRef::try_map(self.block.read(), |raw_block| unsafe {
raw_block.base.add(offset).cast::<T>().as_ref()
})
.expect("The offset should result in a non-null pointer")
}
RcuRef::try_map(table.block.read(), |raw_block| unsafe {
#[inline]
fn lookup_mut(&self, offset: usize) -> RcuRef<RawBlock<T>, UnsafeCell<T>> {
RcuRef::try_map(self.block.read(), |raw_block| unsafe {
raw_block
.base
.add(offset.into())
.add(offset)
.cast_mut()
.cast::<UnsafeCell<T>>()
.as_ref()
@@ -170,109 +245,8 @@ where
}
}
#[derive(Debug)]
pub struct TablePtr<T>(RcuRef<RawBlock<OffsetTableImpl<T>>, UnsafeCell<T>>)
where
OffsetTableImpl<T>: RawBlockTraits;
pub type CodeIndexPtr = TablePtr<IndexPtr>;
pub type F64Ptr = TablePtr<OrderedFloat<f64>>;
impl<T> Clone for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
fn clone(&self) -> Self {
Self(RcuRef::clone(&self.0))
}
}
impl<T: PartialEq> PartialEq for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
fn eq(&self, other: &TablePtr<T>) -> bool {
RcuRef::ptr_eq(&self.0, &other.0) || self.deref() == other.deref()
}
}
impl<T: Eq> Eq for TablePtr<T> where OffsetTableImpl<T>: RawBlockTraits {}
impl<T: PartialOrd + Ord> PartialOrd for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl<T: Ord> Ord for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
(**self).cmp(&**other)
}
}
impl<T: Hash> Hash for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
#[inline(always)]
fn hash<H: Hasher>(&self, hasher: &mut H) {
(self as &T).hash(hasher)
}
}
impl<T: fmt::Display> fmt::Display for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self as &T)
}
}
impl<T> Deref for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
unsafe { self.0.get().as_ref().unwrap() }
}
}
impl<T> DerefMut for TablePtr<T>
where
OffsetTableImpl<T>: RawBlockTraits,
{
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { &mut *self.0.get().as_mut().unwrap() }
}
}
impl<T: 'static> TablePtr<T>
where
OffsetTableImpl<T>: OffsetTable<Stored = T>,
{
#[inline(always)]
pub fn from_offset(offset: <OffsetTableImpl<T> as OffsetTable>::Offset) -> Self {
Self(OffsetTableImpl::<T>::lookup(offset))
}
#[inline(always)]
pub fn as_offset(&self) -> <OffsetTableImpl<T> as OffsetTable>::Offset {
<OffsetTableImpl<T> as OffsetTable>::Offset::from(
self.0.get().addr() - RcuRef::get_root(&self.0).base.addr(),
)
}
}
pub type F64Table = OffsetTableImpl<OrderedFloat<f64>>;
pub type CodeIndexTable = OffsetTableImpl<IndexPtr>;
#[derive(Clone, Copy, Debug)]
pub struct F64Offset(usize);
@@ -284,10 +258,9 @@ impl From<usize> for F64Offset {
}
}
impl Into<usize> for F64Offset {
#[inline(always)]
fn into(self: Self) -> usize {
self.0
impl From<F64Offset> for usize {
fn from(val: F64Offset) -> Self {
val.0
}
}
@@ -301,57 +274,71 @@ impl From<usize> for CodeIndexOffset {
}
}
impl Into<usize> for CodeIndexOffset {
impl From<CodeIndexOffset> for usize {
#[inline(always)]
fn into(self: Self) -> usize {
self.0
fn from(val: CodeIndexOffset) -> Self {
val.0
}
}
impl CodeIndexOffset {
#[inline(always)]
pub fn from_ptr(ptr: CodeIndexPtr) -> Self {
ptr.as_offset()
}
#[inline(always)]
pub fn as_ptr(self) -> CodeIndexPtr {
CodeIndexPtr::from_offset(self)
}
#[inline(always)]
pub fn to_u64(self) -> u64 {
self.0 as u64
}
}
impl PartialEq for CodeIndexOffset {
#[inline(always)]
fn eq(&self, other: &CodeIndexOffset) -> bool {
self.as_ptr() == other.as_ptr()
#[derive(Debug)]
pub struct TablePtr<'a, T: RawBlockTraits>(InnerTablePtr<'a, T>);
#[derive(Debug)]
enum InnerTablePtr<'a, T: RawBlockTraits> {
Concurrent(RcuRef<RawBlock<T>, T>),
Serial(&'a T),
}
impl<T: PartialEq + RawBlockTraits> PartialEq for TablePtr<'_, T> {
fn eq(&self, other: &TablePtr<'_, T>) -> bool {
self.deref() == other.deref()
}
}
impl Eq for CodeIndexOffset {}
impl<T: Eq + RawBlockTraits> Eq for TablePtr<'_, T> {}
impl PartialOrd for CodeIndexOffset {
#[inline(always)]
impl<T: PartialOrd + Ord + RawBlockTraits> PartialOrd for TablePtr<'_, T> {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for CodeIndexOffset {
#[inline(always)]
impl<T: Ord + RawBlockTraits> Ord for TablePtr<'_, T> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.as_ptr().cmp(&other.as_ptr())
(**self).cmp(&**other)
}
}
impl Hash for CodeIndexOffset {
impl<T: Hash + RawBlockTraits> Hash for TablePtr<'_, T> {
#[inline(always)]
fn hash<H: Hasher>(&self, hasher: &mut H) {
self.as_ptr().hash(hasher)
(self as &T).hash(hasher)
}
}
impl<T: fmt::Display + RawBlockTraits> fmt::Display for TablePtr<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self as &T)
}
}
impl<T: RawBlockTraits> Deref for TablePtr<'_, T> {
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
match &self.0 {
InnerTablePtr::Concurrent(rcu_ref) => rcu_ref,
InnerTablePtr::Serial(ref_mut) => ref_mut,
}
}
}
@@ -361,67 +348,103 @@ impl fmt::Display for CodeIndexOffset {
}
}
impl CodeIndexPtr {
#[inline]
pub fn set(&self, val: IndexPtr) {
unsafe { *self.0.get() = val };
}
#[inline]
pub fn replace(&self, val: IndexPtr) -> IndexPtr {
unsafe { self.0.get().replace(val) }
}
}
impl F64Offset {
#[inline(always)]
pub fn from_ptr(ptr: F64Ptr) -> Self {
ptr.as_offset()
}
#[inline(always)]
pub fn as_ptr(self) -> F64Ptr {
F64Ptr::from_offset(self)
}
#[inline(always)]
pub fn to_u64(self) -> u64 {
self.0 as u64
}
}
impl PartialEq for F64Offset {
#[inline(always)]
fn eq(&self, other: &F64Offset) -> bool {
self.as_ptr() == other.as_ptr()
}
}
impl Eq for F64Offset {}
impl PartialOrd for F64Offset {
#[inline(always)]
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for F64Offset {
#[inline(always)]
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.as_ptr().cmp(&other.as_ptr())
}
}
impl Hash for F64Offset {
#[inline(always)]
fn hash<H: Hasher>(&self, hasher: &mut H) {
self.as_ptr().hash(hasher)
}
}
impl fmt::Display for F64Offset {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "F64Offset({})", self.0)
}
}
#[derive(Debug)]
pub struct TablePtrMut<'a, T: RawBlockTraits>(InnerTablePtrMut<'a, T>);
#[derive(Debug)]
enum InnerTablePtrMut<'a, T: RawBlockTraits> {
Concurrent(RcuRef<RawBlock<T>, UnsafeCell<T>>),
Serial(&'a mut T),
}
impl<T: PartialEq + RawBlockTraits> PartialEq for TablePtrMut<'_, T> {
fn eq(&self, other: &TablePtrMut<'_, T>) -> bool {
self.deref() == other.deref()
}
}
impl<T: Eq + RawBlockTraits> Eq for TablePtrMut<'_, T> {}
impl<T: PartialOrd + Ord + RawBlockTraits> PartialOrd for TablePtrMut<'_, T> {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl<T: Ord + RawBlockTraits> Ord for TablePtrMut<'_, T> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
(**self).cmp(&**other)
}
}
impl<T: Hash + RawBlockTraits> Hash for TablePtrMut<'_, T> {
#[inline(always)]
fn hash<H: Hasher>(&self, hasher: &mut H) {
(self as &T).hash(hasher)
}
}
impl<T: fmt::Display + RawBlockTraits> fmt::Display for TablePtrMut<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self as &T)
}
}
impl<T: RawBlockTraits> Deref for TablePtrMut<'_, T> {
type Target = T;
#[inline]
fn deref(&self) -> &Self::Target {
match &self.0 {
InnerTablePtrMut::Concurrent(rcu_ref) => unsafe { rcu_ref.get().as_ref().unwrap() },
InnerTablePtrMut::Serial(ref_mut) => ref_mut,
}
}
}
impl<T: RawBlockTraits> DerefMut for TablePtrMut<'_, T> {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
match &mut self.0 {
InnerTablePtrMut::Concurrent(rcu_ref) => unsafe {
&mut *rcu_ref.get().as_mut().unwrap()
},
InnerTablePtrMut::Serial(ref_mut) => ref_mut,
}
}
}
impl TablePtrMut<'_, IndexPtr> {
#[inline]
pub fn set(&mut self, val: IndexPtr) {
match &mut self.0 {
InnerTablePtrMut::Concurrent(rcu_ref) => unsafe {
*rcu_ref.get() = val;
},
InnerTablePtrMut::Serial(ref_mut) => {
**ref_mut = val;
}
}
}
#[inline]
pub fn replace(&mut self, val: IndexPtr) -> IndexPtr {
match &mut self.0 {
InnerTablePtrMut::Concurrent(rcu_ref) => unsafe { rcu_ref.get().replace(val) },
InnerTablePtrMut::Serial(ref_mut) => mem::replace(*ref_mut, val),
}
}
}