Merge pull request #3266 from Skgland/perf-issue-3265

perf improvements for 3265

src/machine/dispatch.rs

@@ -77,8 +77,6 @@ macro_rules! push_cell {
     }};
 }
 
-static INSTRUCTIONS_PER_INTERRUPT_POLL: usize = 256;
-
 impl MachineState {
     #[inline(always)]
     fn compare(&mut self) -> CallResult {
@@ -1541,8 +1539,13 @@ impl Machine {
     }
 
     fn verify_attr_dispatch_loop(&mut self) -> Option<std::process::ExitCode> {
+        let mut interrupt_counter = std::num::Wrapping(0u8);
         'outer: loop {
-            for _ in 0..INSTRUCTIONS_PER_INTERRUPT_POLL {
+            loop {
+                interrupt_counter += 1;
+                if interrupt_counter.0 == 0 {
+                    break;
+                }
                 match self.code[self.machine_st.p] {
                     Instruction::BreakFromDispatchLoop => {
                         break 'outer;
@@ -1611,9 +1614,20 @@ impl Machine {
     }
 
     pub(super) fn dispatch_loop(&mut self) -> std::process::ExitCode {
+        let mut interrupt_counter = std::num::Wrapping(0u8);
         'outer: loop {
-            for _ in 0..INSTRUCTIONS_PER_INTERRUPT_POLL {
+            loop {
-                match &self.code[self.machine_st.p] {
+                interrupt_counter += 1;
+                if interrupt_counter.0 == 0 {
+                    break;
+                }
+
+                let Some(inst) = self.code.get(self.machine_st.p) else {
+                    // a seperate function marked #[cold] to make the compiler/branch-predictor prefer the happy path
+                    handle_code_index_oob(self.code.len(), self.machine_st.p);
+                };
+
+                match inst {
                     &Instruction::BreakFromDispatchLoop => {
                         break 'outer;
                     }
@@ -4256,12 +4270,12 @@ impl Machine {
                         step_or_fail!(self.machine_st, self.machine_st.p = self.machine_st.cp);
                     }
                     &Instruction::CallInferenceCount => {
-                        let global_count = self.machine_st.cwil.global_count.clone();
+                        let global_count = self.machine_st.cwil.global_count;
                         self.inference_count(self.machine_st.registers[1], global_count);
                         step_or_fail!(self.machine_st, self.machine_st.p += 1);
                     }
                     &Instruction::ExecuteInferenceCount => {
-                        let global_count = self.machine_st.cwil.global_count.clone();
+                        let global_count = self.machine_st.cwil.global_count;
                         self.inference_count(self.machine_st.registers[1], global_count);
                         step_or_fail!(self.machine_st, self.machine_st.p = self.machine_st.cp);
                     }
@@ -6023,3 +6037,9 @@ impl Machine {
         std::process::ExitCode::SUCCESS
     }
 }
+
+#[cold] // this is a seperate function so that we can annotate it as cold
+#[track_caller]
+fn handle_code_index_oob(code_len: usize, p: usize) -> ! {
+    panic!("code pointer p = {p} is oob for code area of size {code_len}");
+}

src/machine/machine_state.rs

@@ -16,8 +16,6 @@ use crate::parser::ast::*;
 use crate::read::TermWriteResult;
 use crate::types::*;
 
-use crate::parser::dashu::Integer;
-
 use indexmap::IndexMap;
 
 use std::convert::TryFrom;
@@ -550,7 +548,8 @@ impl MachineState {
             return true;
         }
 
-        self.cwil.global_count += 1;
+        // use strict_add once msrv is >= 1.91.0
+        self.cwil.global_count = self.cwil.global_count.checked_add(1).unwrap();
 
         if let Some(&(ref limit, block)) = self.cwil.limits.last() {
             if self.cwil.local_count == *limit {
@@ -1112,47 +1111,48 @@ impl MachineState {
 #[allow(clippy::upper_case_acronyms)]
 #[derive(Debug)]
 pub(crate) struct CWIL {
-    local_count: Integer,
+    local_count: u128,
-    pub(crate) global_count: Integer,
+    pub(crate) global_count: u128,
-    limits: Vec<(Integer, usize)>,
+    limits: Vec<(u128, usize)>,
     pub(crate) inference_limit_exceeded: bool,
 }
 
 impl CWIL {
     pub(crate) fn new() -> Self {
         CWIL {
-            local_count: Integer::from(0),
+            local_count: 0,
-            global_count: Integer::from(0),
+            global_count: 0,
             limits: vec![],
             inference_limit_exceeded: false,
         }
     }
 
-    pub(crate) fn add_limit(&mut self, mut limit: Integer, block: usize) -> &Integer {
+    pub(crate) fn add_limit(&mut self, mut limit: u128, block: usize) -> u128 {
-        limit += &self.local_count;
+        // use strict_add once msrv is >= 1.91.0
+        limit = limit.checked_add(self.local_count).unwrap();
 
         match self.limits.last() {
             Some((ref inner_limit, _)) if *inner_limit <= limit => {}
             _ => self.limits.push((limit, block)),
         }
 
-        &self.local_count
+        self.local_count
     }
 
     #[inline(always)]
-    pub(crate) fn remove_limit(&mut self, block: usize) -> &Integer {
+    pub(crate) fn remove_limit(&mut self, block: usize) -> u128 {
         if let Some((_, bl)) = self.limits.last() {
             if bl == &block {
                 self.limits.pop();
             }
         }
 
-        &self.local_count
+        self.local_count
     }
 
     #[inline(always)]
     pub(crate) fn reset(&mut self) {
-        self.local_count = Integer::from(0);
+        self.local_count = 0;
         self.limits.clear();
         self.inference_limit_exceeded = false;
     }

src/machine/mod.rs

@@ -234,12 +234,7 @@ impl Machine {
 
     /// Gets the current inference count.
     pub fn get_inference_count(&mut self) -> u64 {
-        self.machine_st
+        self.machine_st.cwil.global_count.try_into().unwrap()
-            .cwil
-            .global_count
-            .clone()
-            .try_into()
-            .unwrap()
     }
 
     /// Runs the predicate `key` in `module_name` until completion.

src/machine/system_calls.rs

@@ -6141,8 +6141,8 @@ impl Machine {
         let a2 = self.deref_register(2);
 
         let n = match Number::try_from((a2, &self.machine_st.arena.f64_tbl)) {
-            Ok(Number::Fixnum(bp)) => Integer::from(bp.get_num() as usize),
+            Ok(Number::Fixnum(bp)) => bp.get_num() as u128,
-            Ok(Number::Integer(n)) => (*n).clone(),
+            Ok(Number::Integer(n)) => u128::try_from(&*n).unwrap(),
             _ => {
                 let stub = functor_stub(atom!("call_with_inference_limit"), 3);
                 let err = self.machine_st.type_error(ValidType::Integer, a2);
@@ -6153,21 +6153,21 @@ impl Machine {
         let bp = unsafe { a1.to_fixnum_or_cut_point_unchecked() }.get_num() as usize;
         let a3 = self.deref_register(3);
 
-        let count = self.machine_st.cwil.add_limit(n, bp).clone();
+        let count = self.machine_st.cwil.add_limit(n, bp);
         self.inference_count(a3, count);
 
         Ok(())
     }
 
     #[inline(always)]
-    pub(crate) fn inference_count(&mut self, count_var: HeapCellValue, count: Integer) {
+    pub(crate) fn inference_count(&mut self, count_var: HeapCellValue, count: u128) {
-        if let Some(value) = <&Integer as TryInto<i64>>::try_into(&count)
+        if let Some(value) = TryInto::<i64>::try_into(count)
             .ok()
             .and_then(|i| Fixnum::build_with_checked(i).ok())
         {
             self.machine_st.unify_fixnum(value, count_var);
         } else {
-            let count = arena_alloc!(count, &mut self.machine_st.arena);
+            let count = arena_alloc!(Integer::from(count), &mut self.machine_st.arena);
             self.machine_st.unify_big_int(count, count_var);
         }
     }
@@ -6321,11 +6321,11 @@ impl Machine {
         let a2 = self.deref_register(2);
 
         let block = unsafe { a1.to_fixnum_or_cut_point_unchecked() }.get_num() as usize;
-        let count = self.machine_st.cwil.remove_limit(block).clone();
+        let count = self.machine_st.cwil.remove_limit(block);
-        if let Ok(value) = Fixnum::build_with_checked(&count) {
+        if let Ok(value) = Fixnum::build_with_checked(count) {
             self.machine_st.unify_fixnum(value, a2);
         } else {
-            let count = arena_alloc!(count.clone(), &mut self.machine_st.arena);
+            let count = arena_alloc!(Integer::from(count), &mut self.machine_st.arena);
             self.machine_st.unify_big_int(count, a2);
         }
     }

src/parser/ast.rs

@@ -608,6 +608,7 @@ mod private {
     impl<T: FitsInFixnumSeal> MightNotFitInFixnumSeal for T {}
     impl MightNotFitInFixnumSeal for i64 {}
     impl MightNotFitInFixnumSeal for u64 {}
+    impl MightNotFitInFixnumSeal for u128 {}
     impl MightNotFitInFixnumSeal for &Integer {}
     impl MightNotFitInFixnumSeal for Integer {}
     impl MightNotFitInFixnumSeal for usize {}