improve call/N implementation (#1829)

build/instructions_template.rs

@@ -560,8 +560,8 @@ enum SystemClauseType {
     StripModule,
     #[strum_discriminants(strum(props(Arity = "4", Name = "$compile_inline_or_expanded_goal")))]
     CompileInlineOrExpandedGoal,
-    #[strum_discriminants(strum(props(Arity = "arity", Name = "$call_inline")))]
+    #[strum_discriminants(strum(props(Arity = "arity", Name = "$fast_call")))]
-    InlineCallN(usize),
+    FastCallN(usize),
     #[strum_discriminants(strum(props(Arity = "1", Name = "$is_expanded_or_inlined")))]
     IsExpandedOrInlined,
     #[strum_discriminants(strum(props(Arity = "3", Name = "$get_clause_p")))]
@@ -1472,11 +1472,11 @@ fn generate_instruction_preface() -> TokenStream {
                     &Instruction::DefaultExecuteN(arity) => {
                         functor!(atom!("execute_default_n"), [fixnum(arity)])
                     }
-                    &Instruction::CallInlineCallN(arity) => {
+                    &Instruction::CallFastCallN(arity) => {
-                        functor!(atom!("call_n_inline"), [fixnum(arity)])
+                        functor!(atom!("call_fast_call_n"), [fixnum(arity)])
                     }
-                    &Instruction::ExecuteInlineCallN(arity) => {
+                    &Instruction::ExecuteFastCallN(arity) => {
-                        functor!(atom!("call_n_inline"), [fixnum(arity)])
+                        functor!(atom!("execute_fast_call_n"), [fixnum(arity)])
                     }
                     &Instruction::CallTermGreaterThan |
                     &Instruction::CallTermLessThan |

src/machine/dispatch.rs

@@ -5058,12 +5058,12 @@ impl Machine {
                     self.machine_st.fail = !self.is_expanded_or_inlined();
                     step_or_fail!(self, self.machine_st.p = self.machine_st.cp);
                 }
-                &Instruction::CallInlineCallN(arity) => {
+                &Instruction::CallFastCallN(arity) => {
                     let call_at_index = |wam: &mut Machine, name, arity, ptr| {
                         wam.try_call(name, arity, ptr)
                     };
 
-                    try_or_throw!(self.machine_st, self.call_inline(arity, call_at_index));
+                    try_or_throw!(self.machine_st, self.fast_call(arity, call_at_index));
 
                     if self.machine_st.fail {
                         self.machine_st.backtrack();
@@ -5074,12 +5074,12 @@ impl Machine {
                         );
                     }
                 }
-                &Instruction::ExecuteInlineCallN(arity) => {
+                &Instruction::ExecuteFastCallN(arity) => {
                     let call_at_index = |wam: &mut Machine, name, arity, ptr| {
                         wam.try_execute(name, arity, ptr)
                     };
 
-                    try_or_throw!(self.machine_st, self.call_inline(arity, call_at_index));
+                    try_or_throw!(self.machine_st, self.fast_call(arity, call_at_index));
 
                     if self.machine_st.fail {
                         self.machine_st.backtrack();

src/machine/loader.rs

@@ -1698,6 +1698,21 @@ impl Machine {
         let add_clause = || {
             let term = loader.read_term_from_heap(temp_v!(2))?;
 
+            let indexing_arg = match term.name() {
+                Some(atom!(":-")) => term.first_arg().and_then(Term::first_arg),
+                Some(_) => term.first_arg(),
+                None => None,
+            };
+
+            if let Some(indexing_term) = indexing_arg {
+                if let Some(indexing_name) = indexing_term.name() {
+                    loader.wam_prelude
+                        .indices
+                        .goal_expansion_indices
+                        .insert((indexing_name, indexing_term.arity()));
+                }
+            }
+
             loader.incremental_compile_clause(
                 (atom!("goal_expansion"), 2),
                 term,

src/machine/machine_indices.rs

@@ -8,7 +8,7 @@ use crate::machine::machine_state::*;
 use crate::machine::streams::Stream;
 
 use fxhash::FxBuildHasher;
-use indexmap::IndexMap;
+use indexmap::{IndexMap, IndexSet};
 use modular_bitfield::{BitfieldSpecifier, bitfield};
 use modular_bitfield::specifiers::*;
 
@@ -243,12 +243,15 @@ pub(crate) type LocalExtensiblePredicates =
 
 pub(crate) type CodeDir = IndexMap<PredicateKey, CodeIndex, FxBuildHasher>;
 
+pub(crate) type GoalExpansionIndices = IndexSet<PredicateKey, FxBuildHasher>;
+
 #[derive(Debug)]
 pub struct IndexStore {
     pub(super) code_dir: CodeDir,
     pub(super) extensible_predicates: ExtensiblePredicates,
     pub(super) local_extensible_predicates: LocalExtensiblePredicates,
     pub(super) global_variables: GlobalVarDir,
+    pub(super) goal_expansion_indices: GoalExpansionIndices,
     pub(super) meta_predicates: MetaPredicateDir,
     pub(super) modules: ModuleDir,
     pub(super) op_dir: OpDir,
@@ -257,6 +260,11 @@ pub struct IndexStore {
 }
 
 impl IndexStore {
+    #[inline(always)]
+    pub(crate) fn goal_expansion_defined(&self, key: PredicateKey) -> bool {
+        self.goal_expansion_indices.contains(&key)
+    }
+
     pub(crate) fn get_predicate_skeleton_mut(
         &mut self,
         compilation_target: &CompilationTarget,

src/machine/system_calls.rs

@@ -1202,29 +1202,29 @@ impl Machine {
 
     #[inline(always)]
     pub(crate) fn deref_register(&mut self, i: usize) -> HeapCellValue {
-	self.machine_st.store(self.machine_st.deref(self.machine_st.registers[i]))
+	    self.machine_st.store(self.machine_st.deref(self.machine_st.registers[i]))
     }
 
     #[inline(always)]
-    pub(crate) fn call_inline(
+    pub(crate) fn fast_call(
         &mut self,
         arity: usize,
         call_at_index: impl Fn(&mut Machine, Atom, usize, IndexPtr) -> CallResult,
     ) -> CallResult {
         let arity = arity - 1;
-        let goal = self.deref_register(1);
+        let (mut module_name, mut goal) = self.machine_st.strip_module(
+            self.machine_st.registers[1],
+            heap_loc_as_cell!(0),
+        );
 
-        let load_registers = |machine_st: &mut MachineState, goal: HeapCellValue| -> Option<PredicateKey> {
+        let load_registers = |machine_st: &mut MachineState, goal: HeapCellValue, goal_arity: usize| {
             read_heap_cell!(goal,
-                (HeapCellValueTag::Str, s) => {
+                (HeapCellValueTag::Str | HeapCellValueTag::Atom, s) => {
-                    let (name, goal_arity) = cell_as_atom_cell!(machine_st.heap[s])
+                    if goal_arity > 1 {
-                        .get_name_and_arity();
-
-                    if goal_arity > 0 {
                         for idx in (1 .. arity + 1).rev() {
                             machine_st.registers[idx + goal_arity] = machine_st.registers[idx + 1];
                         }
-                    } else {
+                    } else if goal_arity == 0 {
                         for idx in 1 .. arity + 1 {
                             machine_st.registers[idx] = machine_st.registers[idx + 1];
                         }
@@ -1233,8 +1233,6 @@ impl Machine {
                     for idx in 1 .. goal_arity + 1 {
                         machine_st.registers[idx] = machine_st.heap[s+idx];
                     }
-
-                    Some((name, goal_arity))
                 }
                 _ => {
                     unreachable!()
@@ -1242,35 +1240,70 @@ impl Machine {
             )
         };
 
-        read_heap_cell!(goal,
+        let (mut name, mut goal_arity, index_cell_opt) = read_heap_cell!(goal,
             (HeapCellValueTag::Str, s) => {
-                let goal_arity = cell_as_atom_cell!(self.machine_st.heap[s]).get_arity();
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s]).get_name_and_arity();
 
-                if self.machine_st.heap.len() > s + goal_arity + 1 {
+                (name, arity, if self.machine_st.heap.len() > s + arity + 1 {
-                    let index_cell = self.machine_st.heap[s+goal_arity+1];
+                    get_structure_index(self.machine_st.heap[s + arity + 1])
-
+                } else {
-                    if let Some(code_index) = get_structure_index(index_cell) {
+                    None
-                        if code_index.is_undefined() {
+                })
-                            self.machine_st.fail = true;
+            }
-                            return Ok(());
+            (HeapCellValueTag::Atom, (name, arity)) => {
-                        }
+                debug_assert_eq!(arity, 0);
-
+                (name, arity, None)
-                        match load_registers(&mut self.machine_st, goal) {
-                            Some((name, goal_arity)) => {
-                                let arity = goal_arity + arity;
-                                self.machine_st.neck_cut();
-                                return call_at_index(self, name, arity, code_index.get());
-                            }
-                            None => {
-                            }
-                        }
-                    }
-                }
             }
             _ => {
+                self.machine_st.fail = true;
+                return Ok(());
             }
         );
 
+        let mut arity = arity + goal_arity;
+
+        let index_cell = index_cell_opt.or_else(|| {
+            let is_internal_call = name == atom!("$call") && goal_arity > 0;
+
+            if !is_internal_call && self.indices.goal_expansion_defined((name, arity)) {
+                None
+            } else {
+                if is_internal_call {
+                    debug_assert_eq!(goal.get_tag(), HeapCellValueTag::Str);
+                    goal = self.machine_st.heap[goal.get_value()+1];
+                    (module_name, goal) = self.machine_st.strip_module(goal, module_name);
+
+                    if let Some((inner_name, inner_arity)) = self.machine_st.name_and_arity_from_heap(goal) {
+                        arity -= goal_arity;
+                        (name, goal_arity) = (inner_name, inner_arity);
+                        arity += goal_arity;
+                    } else {
+                        return None;
+                    }
+                }
+
+                let module_name = if module_name.get_tag() != HeapCellValueTag::Atom {
+                    if let Some(load_context) = self.load_contexts.last() {
+                        load_context.module
+                    } else {
+                        atom!("user")
+                    }
+                } else {
+                    cell_as_atom!(module_name)
+                };
+
+                self.indices.get_predicate_code_index(name, arity, module_name)
+            }
+        });
+
+        if let Some(code_index) = index_cell {
+            if !code_index.is_undefined() {
+                load_registers(&mut self.machine_st, goal, goal_arity);
+                self.machine_st.neck_cut();
+                return call_at_index(self, name, arity, code_index.get());
+            }
+        }
+
         self.machine_st.fail = true;
         Ok(())
     }
@@ -1489,35 +1522,12 @@ impl Machine {
     }
 
     #[inline(always)]
-    pub(crate) fn prepare_call_clause(&mut self, arity: usize) -> CallResult {
+    pub(crate) fn strip_module(&mut self) {
         let (module_loc, qualified_goal) = self.machine_st.strip_module(
-            self.machine_st.registers[3],
+            self.machine_st.registers[1],
             self.machine_st.registers[2],
         );
 
-        // the first three arguments don't belong to the containing call/N.
-        let arity = arity - 3;
-
-        let (name, narity, s) = self.machine_st.setup_call_n_init_goal_info(
-            qualified_goal,
-            arity,
-        )?;
-
-        let module_loc = self.machine_st.store(self.machine_st.deref(module_loc));
-
-        if module_loc.is_var() {
-            self.load_context_module(module_loc);
-
-            if self.machine_st.fail {
-                self.machine_st.fail = false;
-                self.machine_st.unify_atom(atom!("user"), module_loc);
-
-                if self.machine_st.fail {
-                    return Ok(());
-                }
-            }
-        }
-
         let target_module_loc = self.machine_st.registers[2];
 
         unify_fn!(
@@ -1526,9 +1536,26 @@ impl Machine {
             target_module_loc
         );
 
-        if self.machine_st.fail {
+        let target_qualified_goal = self.machine_st.registers[3];
-            return Ok(());
+
-        }
+        unify_fn!(
+            &mut self.machine_st,
+            qualified_goal,
+            target_qualified_goal
+        );
+    }
+
+    #[inline(always)]
+    pub(crate) fn prepare_call_clause(&mut self, arity: usize) -> CallResult {
+        let qualified_goal = self.deref_register(2);
+
+        // the first two arguments don't belong to the containing call/N.
+        let arity = arity - 2;
+
+        let (name, narity, s) = self.machine_st.setup_call_n_init_goal_info(
+            qualified_goal,
+            arity,
+        )?;
 
         // assemble goal from pre-loaded (narity) and supplementary
         // (arity) arguments.
@@ -1544,15 +1571,10 @@ impl Machine {
             }
 
             for idx in 1 .. arity + 1 {
-                self.machine_st.heap.push(self.machine_st.registers[3 + idx]);
+                self.machine_st.heap.push(self.machine_st.registers[2 + idx]);
             }
 
-            let index_cell = self.machine_st.heap[s + narity + 1];
+            if narity + arity > 0 {
-
-            if get_structure_index(index_cell).is_some() {
-                self.machine_st.heap.push(index_cell);
-                str_loc_as_cell!(h)
-            } else if narity + arity > 0 {
                 str_loc_as_cell!(h)
             } else {
                 heap_loc_as_cell!(h)
@@ -1570,6 +1592,65 @@ impl Machine {
         Ok(())
     }
 
+    #[inline(always)]
+    pub(crate) fn dynamic_module_resolution(
+        &mut self,
+        narity: usize,
+    ) -> Result<(Atom, PredicateKey), MachineStub> {
+        let module_name = self.deref_register(1);
+
+        let module_name = read_heap_cell!(module_name,
+            (HeapCellValueTag::Atom, (name, _arity)) => {
+                debug_assert_eq!(_arity, 0);
+                name
+            }
+            (HeapCellValueTag::Str, s) => {
+                let (module_name, _arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(_arity, 0);
+                module_name
+            }
+            _ if module_name.is_var() => {
+                if let Some(load_context) = self.load_contexts.last() {
+                    load_context.module
+                } else {
+                    atom!("user")
+                }
+            }
+            _ => {
+                unreachable!()
+            }
+        );
+
+        let goal = self.deref_register(2);
+
+        let (name, arity, s) = self.machine_st.setup_call_n_init_goal_info(goal, narity)?;
+
+        // TODO: think we just need the 'Greater' branch here.
+        match arity.cmp(&2) {
+            Ordering::Less => {
+                for i in arity + 1..arity + narity + 1 {
+                    self.machine_st.registers[i] = self.machine_st.registers[i + 2 - arity];
+                }
+            }
+            Ordering::Greater => {
+                for i in (arity + 1..arity + narity + 1).rev() {
+                    self.machine_st.registers[i] = self.machine_st.registers[i + 2 - arity];
+                }
+            }
+            Ordering::Equal => {}
+        }
+
+        let key = (name, arity + narity);
+
+        for i in 1..arity + 1 {
+            self.machine_st.registers[i] = self.machine_st.heap[s + i];
+        }
+
+        Ok((module_name, key))
+    }
+
     #[inline(always)]
     pub(crate) fn is_reset_cont_marker(&self, p: usize) -> bool {
         match &self.code[p] {
@@ -3606,60 +3687,6 @@ impl Machine {
         }
     }
 
-    #[inline(always)]
-    pub(crate) fn dynamic_module_resolution(
-        &mut self,
-        narity: usize,
-    ) -> Result<(Atom, PredicateKey), MachineStub> {
-        let module_name = self.deref_register(1);
-
-        let module_name = read_heap_cell!(module_name,
-            (HeapCellValueTag::Atom, (name, _arity)) => {
-                debug_assert_eq!(_arity, 0);
-                name
-            }
-            (HeapCellValueTag::Str, s) => {
-                let (module_name, _arity) = cell_as_atom_cell!(self.machine_st.heap[s])
-                    .get_name_and_arity();
-
-                debug_assert_eq!(_arity, 0);
-                module_name
-            }
-            _ if module_name.is_var() => {
-                atom!("user")
-            }
-            _ => {
-                unreachable!()
-            }
-        );
-
-        let goal = self.deref_register(2);
-
-        let (name, arity, s) = self.machine_st.setup_call_n_init_goal_info(goal, narity)?;
-
-        match arity.cmp(&2) {
-            Ordering::Less => {
-                for i in arity + 1..arity + narity + 1 {
-                    self.machine_st.registers[i] = self.machine_st.registers[i + 2 - arity];
-                }
-            }
-            Ordering::Greater => {
-                for i in (arity + 1..arity + narity + 1).rev() {
-                    self.machine_st.registers[i] = self.machine_st.registers[i + 2 - arity];
-                }
-            }
-            Ordering::Equal => {}
-        }
-
-        let key = (name, arity + narity);
-
-        for i in 1..arity + 1 {
-            self.machine_st.registers[i] = self.machine_st.heap[s + i];
-        }
-
-        Ok((module_name, key))
-    }
-
     #[inline(always)]
     pub(crate) fn lookup_db_ref(&mut self) {
         let name = cell_as_atom!(self.deref_register(1));

src/macros.rs

@@ -574,6 +574,7 @@ macro_rules! index_store {
             extensible_predicates: ExtensiblePredicates::with_hasher(FxBuildHasher::default()),
             local_extensible_predicates: LocalExtensiblePredicates::with_hasher(FxBuildHasher::default()),
             global_variables: GlobalVarDir::with_hasher(FxBuildHasher::default()),
+            goal_expansion_indices: GoalExpansionIndices::with_hasher(FxBuildHasher::default()),
             meta_predicates: MetaPredicateDir::with_hasher(FxBuildHasher::default()),
             modules: $modules,
             op_dir: $op_dir,

src/toplevel.pl

@@ -181,7 +181,8 @@ submit_query_and_print_results_(Term, VarList) :-
     '$get_b_value'(B),
     bb_put('$report_all', false),
     bb_put('$report_n_more', 0),
-    atts:call_residue_vars(user:Term, AttrVars),
+    expand_goal(Term, user, Term0),
+    atts:call_residue_vars(user:Term0, AttrVars),
     write_eqs_and_read_input(B, VarList, AttrVars),
     !.
 submit_query_and_print_results_(_, _) :-