inline metacalls

src/machine/arithmetic_ops.rs

@@ -390,7 +390,6 @@ pub(crate) fn int_pow(n1: Number, n2: Number, arena: &mut Arena) -> Result<Numbe
 pub(crate) fn pow(n1: Number, n2: Number, culprit: Atom) -> Result<Number, MachineStubGen> {
     if n2.is_negative() && n1.is_zero() {
         let stub_gen = move || functor_stub(culprit, 2);
-
         return Err(undefined_eval_error(stub_gen));
     }
 
@@ -1183,7 +1182,7 @@ impl MachineState {
 
                                 let result = arena_alloc!(
                                     drop_iter_on_err!(self, iter, rdiv(r1, r2)),
-                                    self.arena
+                                    &mut self.arena
                                 );
 
                                 self.interms.push(Number::Rational(result));

src/machine/attributed_variables.pl

@@ -28,11 +28,13 @@ call_verify_attributes([Attr|Attrs], Var, Value, ListOfGoalLists) :-
     sort(Modules0, Modules),
     verify_attrs(Modules, Var, Value, ListOfGoalLists).
 
+error_handler(M, evaluation_error((M:verify_attributes)/3), []).
+% error_handler(_, existence_error(procedure, verify_attributes/3), []).
 
 verify_attrs([Module|Modules], Var, Value, [Module-Goals|ListOfGoalLists]) :-
     catch(Module:verify_attributes(Var, Value, Goals),
-          error(evaluation_error((Module:verify_attributes)/3), verify_attributes/3),
-          Goals = []),
+          error(E, verify_attributes/3),
+          error_handler(Module, E, Goals)),
     verify_attrs(Modules, Var, Value, ListOfGoalLists).
 verify_attrs([], _, _, []).
 

src/machine/compile.rs

@@ -406,7 +406,7 @@ fn merge_indexed_subsequences(
 
                 *o = 0;
 
-                return Some(IndexPtr::Index(outer_threaded_choice_instr_loc + 1));
+                return Some(IndexPtr::index(outer_threaded_choice_instr_loc + 1));
             }
             _ => {}
         },
@@ -785,7 +785,7 @@ fn remove_non_leading_clause(
 
                     *o = 0;
 
-                    Some(IndexPtr::Index(preceding_choice_instr_loc + 1))
+                    Some(IndexPtr::index(preceding_choice_instr_loc + 1))
                 }
                 _ => {
                     unreachable!();
@@ -820,7 +820,7 @@ fn finalize_retract(
             retraction_info,
             &compilation_target,
             key,
-            &code_index,
+            code_index,
             index_ptr,
         );
     }
@@ -849,9 +849,9 @@ fn remove_leading_unindexed_clause(
                     retraction_info,
                 );
 
-                Some(IndexPtr::Index(index_ptr))
+                Some(IndexPtr::index(index_ptr))
             } else {
-                Some(IndexPtr::DynamicUndefined)
+                Some(IndexPtr::dynamic_undefined())
             }
         }
         _ => {
@@ -1131,9 +1131,9 @@ fn prepend_compiled_clause(
     };
 
     if skeleton.core.is_dynamic {
-        IndexPtr::DynamicIndex(clause_loc)
+        IndexPtr::dynamic_index(clause_loc)
     } else {
-        IndexPtr::Index(clause_loc)
+        IndexPtr::index(clause_loc)
     }
 }
 
@@ -1268,9 +1268,9 @@ fn append_compiled_clause(
 
     code_ptr_opt.map(|p| {
         if skeleton.core.is_dynamic {
-            IndexPtr::DynamicIndex(p)
+            IndexPtr::dynamic_index(p)
         } else {
-            IndexPtr::Index(p)
+            IndexPtr::index(p)
         }
     })
 }
@@ -1306,8 +1306,8 @@ fn print_overwrite_warning(
         }
     }
 
-    match code_ptr {
-        IndexPtr::DynamicUndefined | IndexPtr::Undefined => return,
+    match code_ptr.tag() {
+        IndexPtrTag::DynamicUndefined | IndexPtrTag::Undefined => return,
         _ if is_dynamic => return,
         _ => {}
     }
@@ -1471,16 +1471,16 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
         );
 
         let index_ptr = if settings.is_dynamic() {
-            IndexPtr::DynamicIndex(code_ptr)
+            IndexPtr::dynamic_index(code_ptr)
         } else {
-            IndexPtr::Index(code_ptr)
+            IndexPtr::index(code_ptr)
         };
 
         set_code_index(
             &mut self.payload.retraction_info,
             &predicates.compilation_target,
             key,
-            &code_index,
+            code_index,
             index_ptr,
         );
 
@@ -1704,7 +1704,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                         &mut self.payload.retraction_info,
                         &compilation_target,
                         key,
-                        &code_index,
+                        code_index,
                         new_code_ptr,
                     );
                 }
@@ -1745,7 +1745,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                     &mut self.payload.retraction_info,
                     &compilation_target,
                     key,
-                    &code_index,
+                    code_index,
                     new_code_ptr,
                 );
 
@@ -1870,7 +1870,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                                 skeleton.clauses[target_pos].clause_start;
 
                             let index_ptr_opt = if target_pos == 0 {
-                                Some(IndexPtr::Index(clause_loc))
+                                Some(IndexPtr::index(clause_loc))
                             } else {
                                 None
                             };
@@ -2384,13 +2384,15 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                 match self.wam_prelude.indices.modules.get_mut(&filename) {
                     Some(ref mut module) => {
                         let index_ptr = code_index.get();
-                        let code_index = module.code_dir.entry(key).or_insert(code_index);
+                        let code_index = module.code_dir.entry(key)
+                            .or_insert(code_index)
+                            .clone();
 
                         set_code_index(
                             &mut self.payload.retraction_info,
                             &CompilationTarget::Module(filename),
                             key,
-                            &code_index,
+                            code_index,
                             index_ptr,
                         );
                     }
@@ -2418,3 +2420,54 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
         Ok(())
     }
 }
+
+// standalone functions for compiling auxiliary goals used by expand_goal.
+impl Machine {
+    pub(crate) fn get_or_insert_qualified_code_index(
+        &mut self,
+        module_name: HeapCellValue,
+        key: PredicateKey,
+    ) -> CodeIndex {
+        let mut loader: Loader<'_, LiveLoadAndMachineState<'_>> = Loader::new(
+            self,
+            LiveTermStream::new(ListingSource::User),
+        );
+
+        let module_name = if module_name.get_tag() == HeapCellValueTag::Atom {
+            cell_as_atom!(module_name)
+        } else {
+            atom!("user")
+        };
+
+        loader.get_or_insert_qualified_code_index(module_name, key)
+    }
+
+    pub(crate) fn compile_standalone_clause(
+        &mut self,
+        term_loc: RegType,
+        vars: &[Term],
+    ) -> Result<(), SessionError> {
+        let mut compile = || {
+            let mut loader: Loader<'_, LiveLoadAndMachineState<'_>> = Loader::new(
+                self,
+                LiveTermStream::new(ListingSource::User),
+            );
+
+            let term = loader.read_term_from_heap(term_loc)?;
+            let clause = build_rule_body(vars, term);
+
+            let settings = CodeGenSettings {
+                global_clock_tick: None,
+                is_extensible: false,
+                non_counted_bt: true,
+            };
+
+            loader.compile_standalone_clause(clause, settings)
+        };
+
+        let StandaloneCompileResult { clause_code, .. } = compile()?;
+        self.code.extend(clause_code.into_iter());
+
+        Ok(())
+    }
+}

src/machine/copier.rs

@@ -1,4 +1,5 @@
 use crate::atom_table::*;
+use crate::machine::get_structure_index;
 use crate::machine::stack::*;
 use crate::types::*;
 
@@ -248,6 +249,14 @@ impl<T: CopierTarget> CopyTermState<T> {
                     let hcv = self.target[addr + 1 + i];
                     self.target.push(hcv);
                 }
+
+                let index_cell = self.target[addr + 1 + arity];
+
+                if get_structure_index(index_cell).is_some() {
+                    // copy the index pointer trailing this
+                    // inlined or expanded goal.
+                    self.target.push(index_cell);
+                }
             }
             (HeapCellValueTag::Str, h) => {
                 *self.value_at_scan() = str_loc_as_cell!(h);

src/machine/dispatch.rs

@@ -57,18 +57,28 @@ impl MachineState {
         let a2 = self.registers[2];
         let a3 = self.registers[3];
 
+        let check_atom = |machine_st: &mut MachineState, name: Atom, arity: usize| -> Result<(), MachineStub> {
+            match name {
+                atom!(">") | atom!("<") | atom!("=") if arity == 0 => {
+                    Ok(())
+                }
+                _ => {
+                    let err = machine_st.domain_error(DomainErrorType::Order, a1);
+                    Err(machine_st.error_form(err, stub_gen()))
+                }
+            }
+        };
+
         read_heap_cell!(a1,
             (HeapCellValueTag::Atom, (name, arity)) => {
                 debug_assert_eq!(arity, 0);
+                check_atom(self, name, arity)?;
+            }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
 
-                match name {
-                    atom!(">") | atom!("<") | atom!("=") => {
-                    }
-                    _ => {
-                        let err = self.domain_error(DomainErrorType::Order, a1);
-                        return Err(self.error_form(err, stub_gen()));
-                    }
-                }
+                check_atom(self, name, arity)?;
             }
             (HeapCellValueTag::AttrVar | HeapCellValueTag::Var | HeapCellValueTag::StackVar) => {
             }
@@ -360,8 +370,9 @@ impl Machine {
                             debug_assert!(arity == 0);
                             c
                         }
-                        (HeapCellValueTag::Str) => {
-                            s
+                        (HeapCellValueTag::Str, st) => {
+                            let arity = cell_as_atom_cell!(self.machine_st.heap[st]).get_arity();
+                            if arity == 0 { c } else { s }
                         }
                         (HeapCellValueTag::Cons, ptr) => {
                             match ptr.get_tag() {
@@ -421,6 +432,9 @@ impl Machine {
                             debug_assert_eq!(arity, 0);
                             Literal::Atom(atom)
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            Literal::Atom(cell_as_atom_cell!(self.machine_st.heap[s]).get_name())
+                        }
                         (HeapCellValueTag::Cons, cons_ptr) => {
                             match_untyped_arena_ptr!(cons_ptr,
                                 (ArenaHeaderTag::Rational, r) => {
@@ -597,6 +611,7 @@ impl Machine {
                         &mut self.machine_st,
                         try_numeric_result!(sub(n1, n2, &mut self.machine_st.arena), stub_gen)
                     );
+
                     self.machine_st.p += 1;
                 }
                 &Instruction::Mul(ref a1, ref a2, t) => {
@@ -675,7 +690,7 @@ impl Machine {
 
                     self.machine_st.interms[t - 1] = Number::Rational(arena_alloc!(
                         try_or_throw_gen!(&mut self.machine_st, rdiv(r1, r2)),
-                        self.machine_st.arena
+                        &mut self.machine_st.arena
                     ));
 
                     self.machine_st.p += 1;
@@ -688,6 +703,7 @@ impl Machine {
                         &mut self.machine_st,
                         int_floor_div(n1, n2, &mut self.machine_st.arena)
                     );
+
                     self.machine_st.p += 1;
                 }
                 &Instruction::IDiv(ref a1, ref a2, t) => {
@@ -698,6 +714,7 @@ impl Machine {
                         &mut self.machine_st,
                         idiv(n1, n2, &mut self.machine_st.arena)
                     );
+
                     self.machine_st.p += 1;
                 }
                 &Instruction::Abs(ref a1, t) => {
@@ -1122,17 +1139,7 @@ impl Machine {
                     );
                 }
                 &Instruction::NeckCut => {
-                    let b = self.machine_st.b;
-                    let b0 = self.machine_st.b0;
-
-                    if b > b0 {
-                        self.machine_st.b = b0;
-
-                        if b > self.machine_st.e {
-                            self.machine_st.stack.truncate(b);
-                        }
-                    }
-
+                    self.machine_st.neck_cut();
                     self.machine_st.p += 1;
                 }
                 &Instruction::GetLevel(r) => {
@@ -1298,7 +1305,6 @@ impl Machine {
                 }
                 &Instruction::DefaultCallRead(_) => {
                     try_or_throw!(self.machine_st, self.read());
-
                     step_or_fail!(self, self.machine_st.p += 1);
                 }
                 &Instruction::DefaultExecuteRead(_) => {
@@ -2354,6 +2360,16 @@ impl Machine {
                                 self.machine_st.backtrack();
                             }
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            let arity = cell_as_atom_cell!(self.machine_st.heap[s])
+                                .get_arity();
+
+                            if arity == 0 {
+                                self.machine_st.p += 1;
+                            } else {
+                                self.machine_st.backtrack();
+                            }
+                        }
                         (HeapCellValueTag::Char) => {
                             self.machine_st.p += 1;
                         }
@@ -2373,6 +2389,16 @@ impl Machine {
                                 self.machine_st.backtrack();
                             }
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            let arity = cell_as_atom_cell!(self.machine_st.heap[s])
+                                .get_arity();
+
+                            if arity == 0 {
+                                self.machine_st.p = self.machine_st.cp;
+                            } else {
+                                self.machine_st.backtrack();
+                            }
+                        }
                         (HeapCellValueTag::Char) => {
                             self.machine_st.p = self.machine_st.cp;
                         }
@@ -2396,6 +2422,16 @@ impl Machine {
                                 self.machine_st.backtrack();
                             }
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            let arity = cell_as_atom_cell!(self.machine_st.heap[s])
+                                .get_arity();
+
+                            if arity == 0 {
+                                self.machine_st.p += 1;
+                            } else {
+                                self.machine_st.backtrack();
+                            }
+                        }
                         _ => {
                             self.machine_st.backtrack();
                         }
@@ -2416,6 +2452,16 @@ impl Machine {
                                 self.machine_st.backtrack();
                             }
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            let arity = cell_as_atom_cell!(self.machine_st.heap[s])
+                                .get_arity();
+
+                            if arity == 0 {
+                                self.machine_st.p = self.machine_st.cp;
+                            } else {
+                                self.machine_st.backtrack();
+                            }
+                        }
                         _ => {
                             self.machine_st.backtrack();
                         }
@@ -2425,10 +2471,21 @@ impl Machine {
                     let d = self.machine_st.store(self.machine_st.deref(self.machine_st[r]));
 
                     read_heap_cell!(d,
-                        (HeapCellValueTag::Str | HeapCellValueTag::Lis |
-                         HeapCellValueTag::PStrLoc | HeapCellValueTag::CStr) => {
+                        (HeapCellValueTag::Lis |
+                         HeapCellValueTag::PStrLoc |
+                         HeapCellValueTag::CStr) => {
                             self.machine_st.p += 1;
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            let arity = cell_as_atom_cell!(self.machine_st.heap[s])
+                                .get_arity();
+
+                            if arity > 0 {
+                                self.machine_st.p += 1;
+                            } else {
+                                self.machine_st.backtrack();
+                            }
+                        }
                         (HeapCellValueTag::Atom, (_name, arity)) => {
                             if arity > 0 {
                                 self.machine_st.p += 1;
@@ -2445,10 +2502,21 @@ impl Machine {
                     let d = self.machine_st.store(self.machine_st.deref(self.machine_st[r]));
 
                     read_heap_cell!(d,
-                        (HeapCellValueTag::Str | HeapCellValueTag::Lis |
-                         HeapCellValueTag::PStrLoc | HeapCellValueTag::CStr) => {
+                        (HeapCellValueTag::Lis |
+                         HeapCellValueTag::PStrLoc |
+                         HeapCellValueTag::CStr) => {
                             self.machine_st.p = self.machine_st.cp;
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            let arity = cell_as_atom_cell!(self.machine_st.heap[s])
+                                .get_arity();
+
+                            if arity > 0 {
+                                self.machine_st.p = self.machine_st.cp;
+                            } else {
+                                self.machine_st.backtrack();
+                            }
+                        }
                         (HeapCellValueTag::Atom, (_name, arity)) => {
                             if arity > 0 {
                                 self.machine_st.p = self.machine_st.cp;
@@ -3500,7 +3568,10 @@ impl Machine {
                         self.dynamic_module_resolution(arity - 2)
                     );
 
-                    try_or_throw!(self.machine_st, self.call_clause(module_name, key));
+                    try_or_throw!(
+                        self.machine_st,
+                        self.call_clause(module_name, key)
+                    );
 
                     if self.machine_st.fail {
                         self.machine_st.backtrack();
@@ -3512,7 +3583,10 @@ impl Machine {
                         self.dynamic_module_resolution(arity - 2)
                     );
 
-                    try_or_throw!(self.machine_st, self.execute_clause(module_name, key));
+                    try_or_throw!(
+                        self.machine_st,
+                        self.execute_clause(module_name, key)
+                    );
 
                     if self.machine_st.fail {
                         self.machine_st.backtrack();
@@ -4886,7 +4960,7 @@ impl Machine {
                 &Instruction::ExecuteStripModule(_) => {
                     let (module_loc, qualified_goal) = self.machine_st.strip_module(
                         self.machine_st.registers[1],
-                        self.machine_st.registers[2]
+                        self.machine_st.registers[2],
                     );
 
                     let target_module_loc = self.machine_st.registers[2];
@@ -4915,6 +4989,44 @@ impl Machine {
                     try_or_throw!(self.machine_st, self.prepare_call_clause(arity));
                     step_or_fail!(self, self.machine_st.p = self.machine_st.cp);
                 }
+                &Instruction::CallCompileInlineOrExpandedGoal(_) => {
+                    try_or_throw!(self.machine_st, self.compile_inline_or_expanded_goal());
+                    step_or_fail!(self, self.machine_st.p += 1);
+                }
+                &Instruction::ExecuteCompileInlineOrExpandedGoal(_) => {
+                    try_or_throw!(self.machine_st, self.compile_inline_or_expanded_goal());
+                    step_or_fail!(self, self.machine_st.p = self.machine_st.cp);
+                }
+                &Instruction::CallIsExpandedOrInlined(_) => {
+                    self.is_expanded_or_inlined();
+                    step_or_fail!(self, self.machine_st.p += 1);
+                }
+                &Instruction::ExecuteIsExpandedOrInlined(_) => {
+                    self.is_expanded_or_inlined();
+                    step_or_fail!(self, self.machine_st.p = self.machine_st.cp);
+                }
+                &Instruction::CallInlineCallN(arity, _) => {
+                    let call_at_index = |wam: &mut Machine, name, arity, ptr| {
+                        wam.try_call(name, arity, ptr)
+                    };
+
+                    try_or_throw!(self.machine_st, self.call_inline(arity, call_at_index));
+
+                    if self.machine_st.fail {
+                        self.machine_st.backtrack();
+                    }
+                }
+                &Instruction::ExecuteInlineCallN(arity, _) => {
+                    let call_at_index = |wam: &mut Machine, name, arity, ptr| {
+                        wam.try_execute(name, arity, ptr)
+                    };
+
+                    try_or_throw!(self.machine_st, self.call_inline(arity, call_at_index));
+
+                    if self.machine_st.fail {
+                        self.machine_st.backtrack();
+                    }
+                }
             }
         }
 

src/machine/gc.rs

@@ -2,6 +2,9 @@ use crate::atom_table::*;
 use crate::machine::heap::*;
 use crate::types::*;
 
+#[cfg(test)]
+use crate::heap_iter::FocusedHeapIter;
+
 use core::marker::PhantomData;
 
 pub(crate) trait UnmarkPolicy {
@@ -69,6 +72,14 @@ pub(crate) struct StacklessPreOrderHeapIter<'a, UMP: UnmarkPolicy> {
     _marker: PhantomData<UMP>,
 }
 
+#[cfg(test)]
+impl<'a> FocusedHeapIter for StacklessPreOrderHeapIter<'a, IteratorUMP> {
+    #[inline]
+    fn focus(&self) -> usize {
+        self.current
+    }
+}
+
 impl<'a, UMP: UnmarkPolicy> Drop for StacklessPreOrderHeapIter<'a, UMP> {
     fn drop(&mut self) {
         if self.current == self.start {

src/machine/heap.rs

@@ -1,6 +1,7 @@
 use crate::arena::*;
 use crate::atom_table::*;
 use crate::forms::*;
+use crate::machine::machine_indices::*;
 use crate::machine::partial_string::*;
 use crate::parser::ast::*;
 use crate::types::*;
@@ -17,6 +18,9 @@ impl From<Literal> for HeapCellValue {
         match literal {
             Literal::Atom(name) => atom_as_cell!(name),
             Literal::Char(c) => char_as_cell!(c),
+            Literal::CodeIndex(ptr) => {
+                untyped_arena_ptr_as_cell!(UntypedArenaPtr::from(ptr))
+            }
             Literal::Fixnum(n) => fixnum_as_cell!(n),
             Literal::Integer(bigint_ptr) => {
                 typed_arena_ptr_as_cell!(bigint_ptr)
@@ -65,6 +69,9 @@ impl TryFrom<HeapCellValue> for Literal {
                      (ArenaHeaderTag::Rational, n) => {
                          Ok(Literal::Rational(n))
                      }
+                     (ArenaHeaderTag::IndexPtr, _ip) => {
+                         Ok(Literal::CodeIndex(CodeIndex::from(cons_ptr)))
+                     }
                      _ => {
                          Err(())
                      }

src/machine/load_state.rs

@@ -21,12 +21,12 @@ pub(super) fn set_code_index(
     retraction_info: &mut RetractionInfo,
     compilation_target: &CompilationTarget,
     key: PredicateKey,
-    code_index: &CodeIndex,
+    mut code_index: CodeIndex,
     code_ptr: IndexPtr,
 ) {
     let record = match compilation_target {
         CompilationTarget::User => {
-            if IndexPtr::Undefined == code_index.get() {
+            if IndexPtrTag::Undefined == code_index.get().tag() {
                 code_index.set(code_ptr);
                 RetractionRecord::AddedUserPredicate(key)
             } else {
@@ -35,7 +35,7 @@ pub(super) fn set_code_index(
             }
         }
         CompilationTarget::Module(ref module_name) => {
-            if IndexPtr::Undefined == code_index.get() {
+            if IndexPtrTag::Undefined == code_index.get().tag() {
                 code_index.set(code_ptr);
                 RetractionRecord::AddedModulePredicate(*module_name, key)
             } else {
@@ -48,12 +48,10 @@ pub(super) fn set_code_index(
     retraction_info.push_record(record);
 }
 
-fn add_op_decl_as_module_export(
+fn add_op_decl_as_module_export<'a, LS: LoadState<'a>>(
+    payload: &mut LS::LoaderFieldType,
     module_op_dir: &mut OpDir,
-    compilation_target: &CompilationTarget,
-    retraction_info: &mut RetractionInfo,
     wam_op_dir: &mut OpDir,
-    module_op_exports: &mut ModuleOpExports,
     op_decl: &OpDecl,
 ) {
     /*
@@ -65,20 +63,21 @@ fn add_op_decl_as_module_export(
 
     match op_decl.insert_into_op_dir(wam_op_dir) {
         Some(op_desc) => {
-            retraction_info.push_record(RetractionRecord::ReplacedUserOp(
+            payload.retraction_info.push_record(RetractionRecord::ReplacedUserOp(
                 *op_decl,
                 op_desc,
             ));
 
-            module_op_exports.push((*op_decl, Some(op_desc)));
+            payload.module_op_exports.push((*op_decl, Some(op_desc)));
         }
         None => {
-            retraction_info.push_record(RetractionRecord::AddedUserOp(*op_decl));
-            module_op_exports.push((*op_decl, None));
+            payload.retraction_info.push_record(RetractionRecord::AddedUserOp(*op_decl));
+            payload.module_op_exports.push((*op_decl, None));
         }
     }
 
-    add_op_decl(retraction_info, compilation_target, module_op_dir, op_decl);
+    let compilation_target = payload.compilation_target;
+    add_op_decl(&mut payload.retraction_info, &compilation_target, module_op_dir, op_decl);
 }
 
 pub(super) fn add_op_decl(
@@ -117,8 +116,8 @@ pub(super) fn add_op_decl(
     }
 }
 
-pub(super) fn import_module_exports(
-    retraction_info: &mut RetractionInfo,
+pub(super) fn import_module_exports<'a, LS: LoadState<'a>>(
+    payload: &mut LS::LoaderFieldType,
     compilation_target: &CompilationTarget,
     imported_module: &Module,
     code_dir: &mut CodeDir,
@@ -135,16 +134,18 @@ pub(super) fn import_module_exports(
                 }
 
                 if let Some(src_code_index) = imported_module.code_dir.get(&key) {
+                    let arena = &mut LS::machine_st(payload).arena;
+
                     let target_code_index = code_dir
                         .entry(key)
-                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .or_insert_with(|| CodeIndex::default(arena))
                         .clone();
 
                     set_code_index(
-                        retraction_info,
+                        &mut payload.retraction_info,
                         compilation_target,
                         key,
-                        &target_code_index,
+                        target_code_index,
                         src_code_index.get(),
                     );
                 } else {
@@ -155,7 +156,7 @@ pub(super) fn import_module_exports(
                 }
             }
             ModuleExport::OpDecl(ref op_decl) => {
-                add_op_decl(retraction_info, compilation_target, op_dir, op_decl);
+                add_op_decl(&mut payload.retraction_info, compilation_target, op_dir, op_decl);
             }
         }
     }
@@ -163,15 +164,14 @@ pub(super) fn import_module_exports(
     Ok(())
 }
 
-fn import_module_exports_into_module(
-    retraction_info: &mut RetractionInfo,
+fn import_module_exports_into_module<'a, LS: LoadState<'a>>(
+    payload: &mut LS::LoaderFieldType,
     compilation_target: &CompilationTarget,
     imported_module: &Module,
     code_dir: &mut CodeDir,
     op_dir: &mut OpDir,
     meta_predicates: &mut MetaPredicateDir,
     wam_op_dir: &mut OpDir,
-    module_op_exports: &mut ModuleOpExports,
 ) -> Result<(), SessionError> {
     for export in imported_module.module_decl.exports.iter() {
         match export {
@@ -183,16 +183,18 @@ fn import_module_exports_into_module(
                 }
 
                 if let Some(src_code_index) = imported_module.code_dir.get(&key) {
+                    let arena = &mut LS::machine_st(payload).arena;
+
                     let target_code_index = code_dir
                         .entry(key)
-                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .or_insert_with(|| CodeIndex::default(arena))
                         .clone();
 
                     set_code_index(
-                        retraction_info,
+                        &mut payload.retraction_info,
                         compilation_target,
                         key,
-                        &target_code_index,
+                        target_code_index,
                         src_code_index.get(),
                     );
                 } else {
@@ -203,12 +205,10 @@ fn import_module_exports_into_module(
                 }
             }
             ModuleExport::OpDecl(ref op_decl) => {
-                add_op_decl_as_module_export(
+                add_op_decl_as_module_export::<LS>(
+                    payload,
                     op_dir,
-                    compilation_target,
-                    retraction_info,
                     wam_op_dir,
-                    module_op_exports,
                     op_decl,
                 );
             }
@@ -218,14 +218,12 @@ fn import_module_exports_into_module(
     Ok(())
 }
 
-fn import_qualified_module_exports(
-    retraction_info: &mut RetractionInfo,
+fn import_qualified_module_exports<'a, LS: LoadState<'a>>(
+    payload: &mut LS::LoaderFieldType,
     compilation_target: &CompilationTarget,
     imported_module: &Module,
     exports: &IndexSet<ModuleExport>,
-    code_dir: &mut CodeDir,
-    op_dir: &mut OpDir,
-    meta_predicates: &mut MetaPredicateDir,
+    wam_prelude: &mut MachinePreludeView,
 ) -> Result<(), SessionError> {
     for export in imported_module.module_decl.exports.iter() {
         if !exports.contains(export) {
@@ -237,20 +235,22 @@ fn import_qualified_module_exports(
                 let key = (*name, *arity);
 
                 if let Some(meta_specs) = imported_module.meta_predicates.get(&key) {
-                    meta_predicates.insert(key.clone(), meta_specs.clone());
+                    wam_prelude.indices.meta_predicates.insert(key.clone(), meta_specs.clone());
                 }
 
                 if let Some(src_code_index) = imported_module.code_dir.get(&key) {
-                    let target_code_index = code_dir
+                    let arena = &mut LS::machine_st(payload).arena;
+
+                    let target_code_index = wam_prelude.indices.code_dir
                         .entry(key.clone())
-                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .or_insert_with(|| CodeIndex::new(IndexPtr::undefined(), arena))
                         .clone();
 
                     set_code_index(
-                        retraction_info,
+                        &mut payload.retraction_info,
                         compilation_target,
                         key,
-                        &target_code_index,
+                        target_code_index,
                         src_code_index.get(),
                     );
                 } else {
@@ -261,7 +261,12 @@ fn import_qualified_module_exports(
                 }
             }
             ModuleExport::OpDecl(ref op_decl) => {
-                add_op_decl(retraction_info, compilation_target, op_dir, op_decl);
+                add_op_decl(
+                    &mut payload.retraction_info,
+                    compilation_target,
+                    &mut wam_prelude.indices.op_dir,
+                    op_decl,
+                );
             }
         }
     }
@@ -269,17 +274,17 @@ fn import_qualified_module_exports(
     Ok(())
 }
 
-fn import_qualified_module_exports_into_module(
-    retraction_info: &mut RetractionInfo,
-    compilation_target: &CompilationTarget,
+fn import_qualified_module_exports_into_module<'a, LS: LoadState<'a>>(
+    payload: &mut LS::LoaderFieldType,
     imported_module: &Module,
     exports: &IndexSet<ModuleExport>,
     code_dir: &mut CodeDir,
     op_dir: &mut OpDir,
     meta_predicates: &mut MetaPredicateDir,
     wam_op_dir: &mut OpDir,
-    module_op_exports: &mut ModuleOpExports,
 ) -> Result<(), SessionError> {
+    let payload_compilation_target = payload.compilation_target;
+
     for export in imported_module.module_decl.exports.iter() {
         if !exports.contains(export) {
             continue;
@@ -294,16 +299,18 @@ fn import_qualified_module_exports_into_module(
                 }
 
                 if let Some(src_code_index) = imported_module.code_dir.get(&key) {
+                    let arena = &mut LS::machine_st(payload).arena;
+
                     let target_code_index = code_dir
                         .entry(key)
-                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .or_insert_with(|| CodeIndex::new(IndexPtr::undefined(), arena))
                         .clone();
 
                     set_code_index(
-                        retraction_info,
-                        compilation_target,
+                        &mut payload.retraction_info,
+                        &payload_compilation_target,
                         key,
-                        &target_code_index,
+                        target_code_index,
                         src_code_index.get(),
                     );
                 } else {
@@ -314,12 +321,10 @@ fn import_qualified_module_exports_into_module(
                 }
             }
             ModuleExport::OpDecl(ref op_decl) => {
-                add_op_decl_as_module_export(
+                add_op_decl_as_module_export::<LS>(
+                    payload,
                     op_dir,
-                    compilation_target,
-                    retraction_info,
                     wam_op_dir,
-                    module_op_exports,
                     op_decl,
                 );
             }
@@ -464,7 +469,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
             None => return,
         };
 
-        for (key, code_index) in &removed_module.code_dir {
+        for (key, code_index) in removed_module.code_dir.iter_mut() {
             match removed_module
                 .local_extensible_predicates
                 .get(&(CompilationTarget::User, *key))
@@ -473,7 +478,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                 _ => {}
             }
 
-            let old_index_ptr = code_index.replace(IndexPtr::Undefined);
+            let old_index_ptr = code_index.replace(IndexPtr::undefined());
 
             self.payload.retraction_info
                 .push_record(RetractionRecord::ReplacedModulePredicate(
@@ -512,11 +517,11 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
             for export in removed_module.module_decl.exports.iter() {
                 match export {
                     ModuleExport::PredicateKey(ref key) => {
-                        match (removed_module.code_dir.get(key), code_dir.get(key)) {
+                        match (removed_module.code_dir.get(key), code_dir.get_mut(key)) {
                             (Some(module_code_index), Some(target_code_index))
                                 if module_code_index.get() == target_code_index.get() =>
                             {
-                                let old_index_ptr = target_code_index.replace(IndexPtr::Undefined);
+                                let old_index_ptr = target_code_index.replace(IndexPtr::undefined());
                                 retraction_info.push_record(predicate_retractor(*key, old_index_ptr));
                             }
                             _ => {}
@@ -584,7 +589,10 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
             Some(ref mut module) => module
                 .code_dir
                 .entry(key)
-                .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                .or_insert_with(|| CodeIndex::new(
+                    IndexPtr::undefined(),
+                    &mut LS::machine_st(&mut self.payload).arena,
+                ))
                 .clone(),
             None => {
                 self.add_dynamically_generated_module(module_name);
@@ -593,7 +601,10 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                     Some(ref mut module) => module
                         .code_dir
                         .entry(key)
-                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .or_insert_with(|| CodeIndex::new(
+                            IndexPtr::undefined(),
+                            &mut LS::machine_st(&mut self.payload).arena,
+                        ))
                         .clone(),
                     None => {
                         unreachable!()
@@ -608,13 +619,15 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
         key: PredicateKey,
         compilation_target: CompilationTarget,
     ) -> CodeIndex {
+        let arena = &mut LS::machine_st(&mut self.payload).arena;
+
         match compilation_target {
             CompilationTarget::User => self
                 .wam_prelude
                 .indices
                 .code_dir
                 .entry(key)
-                .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                .or_insert_with(|| CodeIndex::new(IndexPtr::undefined(), arena))
                 .clone(),
             CompilationTarget::Module(module_name) => {
                 self.get_or_insert_local_code_index(module_name, key)
@@ -627,13 +640,15 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
         module_name: Atom,
         key: PredicateKey,
     ) -> CodeIndex {
+        let arena = &mut LS::machine_st(&mut self.payload).arena;
+
         if module_name == atom!("user") {
             return self
                 .wam_prelude
                 .indices
                 .code_dir
                 .entry(key)
-                .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                .or_insert_with(|| CodeIndex::new(IndexPtr::undefined(), arena))
                 .clone();
         } else {
             self.get_or_insert_local_code_index(module_name, key)
@@ -732,14 +747,10 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
             CompilationTarget::Module(ref module_name) => {
                 match self.wam_prelude.indices.modules.get_mut(module_name) {
                     Some(ref mut module) => {
-                        let payload: &mut LoadStatePayload<_> = &mut self.payload;
-
-                        add_op_decl_as_module_export(
+                        add_op_decl_as_module_export::<LS>(
+                            &mut self.payload,
                             &mut module.op_dir,
-                            &payload.compilation_target,
-                            &mut payload.retraction_info,
                             &mut self.wam_prelude.indices.op_dir,
-                            &mut payload.module_op_exports,
                             op_decl,
                         );
                     }
@@ -752,7 +763,9 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
     }
 
     pub(super) fn get_clause_type(&mut self, name: Atom, arity: usize) -> ClauseType {
-        match ClauseType::from(name, arity) {
+        let arena = &mut LS::machine_st(&mut self.payload).arena;
+
+        match ClauseType::from(name, arity, arena) {
             ClauseType::Named(arity, name, _) => {
                 let payload_compilation_target = self.payload.compilation_target;
 
@@ -773,7 +786,9 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
         name: Atom,
         arity: usize,
     ) -> ClauseType {
-        match ClauseType::from(name, arity) {
+        let arena = &mut LS::machine_st(&mut self.payload).arena;
+
+        match ClauseType::from(name, arity, arena) {
             ClauseType::Named(arity, name, _) => {
                 let key = (name, arity);
                 let idx = self.get_or_insert_qualified_code_index(module_name, key);
@@ -784,6 +799,16 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
         }
     }
 
+    pub(super) fn get_meta_specs(&self, name: Atom, arity: usize) -> Option<&Vec<MetaSpec>> {
+        self.wam_prelude
+            .indices
+            .get_meta_predicate_spec(
+                name,
+                arity,
+                &self.payload.compilation_target,
+            )
+    }
+
     pub(super) fn add_meta_predicate_record(
         &mut self,
         module_name: Atom,
@@ -894,8 +919,8 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                 return;
             }
 
-            import_module_exports(
-                &mut self.payload.retraction_info,
+            import_module_exports::<LS>(
+                &mut self.payload,
                 &module_compilation_target,
                 builtins,
                 code_dir,
@@ -992,14 +1017,10 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
 
         for export in &module.module_decl.exports {
             if let ModuleExport::OpDecl(ref op_decl) = export {
-                let payload: &mut LoadStatePayload<_> = &mut self.payload;
-
-                add_op_decl_as_module_export(
+                add_op_decl_as_module_export::<LS>(
+                    &mut self.payload,
                     &mut module.op_dir,
-                    &payload.compilation_target, // this is a Module.
-                    &mut payload.retraction_info,
                     &mut self.wam_prelude.indices.op_dir,
-                    &mut payload.module_op_exports,
                     op_decl,
                 );
             }
@@ -1018,8 +1039,8 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
 
             match &payload_compilation_target {
                 CompilationTarget::User => {
-                    import_module_exports(
-                        &mut self.payload.retraction_info,
+                    import_module_exports::<LS>(
+                        &mut self.payload,
                         &payload_compilation_target,
                         &module,
                         &mut self.wam_prelude.indices.code_dir,
@@ -1030,17 +1051,14 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                 CompilationTarget::Module(ref defining_module_name) => {
                     match self.wam_prelude.indices.modules.get_mut(defining_module_name) {
                         Some(ref mut target_module) => {
-                            let payload: &mut LoadStatePayload<_> = &mut self.payload;
-
-                            import_module_exports_into_module(
-                                &mut payload.retraction_info,
+                            import_module_exports_into_module::<LS>(
+                                &mut self.payload,
                                 &payload_compilation_target,
                                 &module,
                                 &mut target_module.code_dir,
                                 &mut target_module.op_dir,
                                 &mut target_module.meta_predicates,
                                 &mut self.wam_prelude.indices.op_dir,
-                                &mut payload.module_op_exports,
                             )?;
                         }
                         None => {
@@ -1070,47 +1088,38 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
         if let Some(module) = self.wam_prelude.indices.modules.remove(&module_name) {
             let payload_compilation_target = self.payload.compilation_target;
 
-            match &payload_compilation_target {
+            let result = match &payload_compilation_target {
                 CompilationTarget::User => {
-                    import_qualified_module_exports(
-                        &mut self.payload.retraction_info,
+                    import_qualified_module_exports::<LS>(
+                        &mut self.payload,
                         &payload_compilation_target,
                         &module,
                         &exports,
-                        &mut self.wam_prelude.indices.code_dir,
-                        &mut self.wam_prelude.indices.op_dir,
-                        &mut self.wam_prelude.indices.meta_predicates,
-                    )?;
+                        &mut self.wam_prelude,
+                    )
                 }
                 CompilationTarget::Module(ref defining_module_name) => {
                     match self.wam_prelude.indices.modules.get_mut(defining_module_name) {
                         Some(ref mut target_module) => {
-                            let payload: &mut LoadStatePayload<_> = &mut self.payload;
-
-                            import_qualified_module_exports_into_module(
-                                &mut payload.retraction_info,
-                                &payload_compilation_target,
+                            import_qualified_module_exports_into_module::<LS>(
+                                &mut self.payload,
                                 &module,
                                 &exports,
                                 &mut target_module.code_dir,
                                 &mut target_module.op_dir,
                                 &mut target_module.meta_predicates,
                                 &mut self.wam_prelude.indices.op_dir,
-                                &mut payload.module_op_exports,
-                            )?;
+                            )
                         }
                         None => {
-                            // we find ourselves here because we're trying to import
-                            // a module into itself as it is being defined.
-                            self.wam_prelude.indices.modules.insert(module_name, module);
-                            return Err(SessionError::ModuleCannotImportSelf(module_name));
+                            Err(SessionError::ModuleCannotImportSelf(module_name))
                         }
                     }
                 }
-            }
+            };
 
             self.wam_prelude.indices.modules.insert(module_name, module);
-            Ok(())
+            result
         } else {
             Err(SessionError::ExistenceError(ExistenceError::Module(module_name)))
         }
@@ -1168,7 +1177,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                     indices: self.wam_prelude.indices,
                     code: self.wam_prelude.code,
                     load_contexts: self.wam_prelude.load_contexts,
-                }
+                },
             };
 
             subloader.load()?
@@ -1231,7 +1240,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                     indices: self.wam_prelude.indices,
                     code: self.wam_prelude.code,
                     load_contexts: self.wam_prelude.load_contexts,
-                }
+                },
             };
 
             subloader.load()?

src/machine/loader.rs

@@ -510,6 +510,21 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                     term_stack.push(Term::Literal(Cell::default(), Literal::try_from(addr).unwrap()));
                 }
                 (HeapCellValueTag::Atom, (name, arity)) => {
+                    let h = iter.focus();
+                    let mut arity = arity;
+
+                    if iter.heap.len() > h + arity + 1 {
+                        let value = iter.heap[h + arity + 1];
+
+                        if let Some(idx) = get_structure_index(value) {
+                            term_stack.push(
+                                Term::Literal(Cell::default(), Literal::CodeIndex(idx))
+                            );
+
+                            arity += 1;
+                        }
+                    }
+
                     if arity == 0 {
                         term_stack.push(Term::Literal(Cell::default(), Literal::Atom(name)));
                     } else {
@@ -708,7 +723,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                             module
                                 .code_dir
                                 .get_mut(&key)
-                                .map(|code_idx| code_idx.replace(old_code_idx));
+                                .map(|code_idx| code_idx.set(old_code_idx));
                         }
                         None => {}
                     }
@@ -733,7 +748,7 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
                         .indices
                         .code_dir
                         .get_mut(&key)
-                        .map(|code_idx| code_idx.replace(old_code_idx));
+                        .map(|code_idx| code_idx.set(old_code_idx));
                 }
                 RetractionRecord::AddedIndex(index_key, clause_loc) => {
                     // WAS: inner_index_locs) => {
@@ -1271,13 +1286,13 @@ impl<'a, LS: LoadState<'a>> Loader<'a, LS> {
 
         let code_index = self.get_or_insert_code_index(key, compilation_target);
 
-        if let IndexPtr::Undefined = code_index.get() {
+        if code_index.is_undefined() {
             set_code_index(
                 &mut self.payload.retraction_info,
                 &compilation_target,
                 key,
-                &code_index,
-                IndexPtr::DynamicUndefined,
+                code_index,
+                IndexPtr::dynamic_undefined(),
             );
         }
     }
@@ -1491,7 +1506,6 @@ impl Machine {
         let mut loader = self.loader_from_heap_evacuable(temp_v!(3));
 
         let declare_module = || {
-            // let export_list = export_list?;
             let exports = loader.extract_module_export_list_from_heap(temp_v!(2))?;
 
             let module_decl = ModuleDecl {
@@ -2019,10 +2033,10 @@ impl Machine {
                 .indices
                 .remove_predicate_skeleton(&compilation_target, &key);
 
-            let code_index = loader
+            let mut code_index = loader
                 .get_or_insert_code_index(key, compilation_target);
 
-            code_index.set(IndexPtr::Undefined);
+            code_index.set(IndexPtr::undefined());
 
             loader.payload.compilation_target = clause_clause_compilation_target;
 
@@ -2187,7 +2201,7 @@ impl Machine {
 
         let (predicate_name, arity) = self
             .machine_st
-            .read_predicate_key(self.machine_st[temp_v!(2)], self.machine_st[temp_v!(3)]);
+            .read_predicate_key(self.machine_st.registers[2], self.machine_st.registers[3]);
 
         let compilation_target = match module_name {
             atom!("user") => CompilationTarget::User,
@@ -2313,7 +2327,7 @@ impl Machine {
             .machine_st
             .read_predicate_key(self.machine_st.registers[1], self.machine_st.registers[2]);
 
-        match ClauseType::from(key.0, key.1) {
+        match ClauseType::from(key.0, key.1, &mut self.machine_st.arena) {
             ClauseType::BuiltIn(_) | ClauseType::Inlined(..) | ClauseType::CallN(_) => {
                 return;
             }
@@ -2355,14 +2369,19 @@ impl<'a> Loader<'a, LiveLoadAndMachineState<'a>> {
 
 #[inline]
 pub(super) fn load_module(
+    machine_st: &mut MachineState,
     code_dir: &mut CodeDir,
     op_dir: &mut OpDir,
     meta_predicate_dir: &mut MetaPredicateDir,
     compilation_target: &CompilationTarget,
     module: &Module,
 ) {
-    import_module_exports(
-        &mut RetractionInfo::new(0),
+    let ts = LiveTermStream::new(ListingSource::User);
+    let payload = LoadStatePayload::new(0, ts);
+    let mut payload = LiveLoadAndMachineState::new(machine_st, payload);
+
+    import_module_exports::<LiveLoadAndMachineState>(
+        &mut payload,
         &compilation_target,
         module,
         code_dir,

src/machine/machine_indices.rs

@@ -4,18 +4,18 @@ use crate::arena::*;
 use crate::atom_table::*;
 use crate::fixtures::*;
 use crate::forms::*;
-use crate::instructions::*;
 use crate::machine::loader::*;
 use crate::machine::machine_state::*;
 use crate::machine::streams::Stream;
 
 use fxhash::FxBuildHasher;
 use indexmap::IndexMap;
+use modular_bitfield::{BitfieldSpecifier, bitfield};
+use modular_bitfield::specifiers::*;
 
-use std::cell::Cell;
 use std::cmp::Ordering;
 use std::collections::BTreeSet;
-use std::ops::Deref;
+use std::ops::{Deref, DerefMut};
 use std::rc::Rc;
 
 use crate::types::*;
@@ -59,9 +59,6 @@ impl PartialOrd<Ref> for HeapCellValue {
                 }
             }
             (HeapCellValueTag::AttrVar | HeapCellValueTag::Var, h1) => {
-            // _ if self.is_ref() => {
-            //     let h1 = self.get_value();
-
                 match r.get_tag() {
                     RefTag::StackCell => Some(Ordering::Less),
                     _ => {
@@ -77,52 +74,133 @@ impl PartialOrd<Ref> for HeapCellValue {
     }
 }
 
-#[derive(Debug, Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)]
-pub enum IndexPtr {
-    DynamicUndefined, // a predicate, declared as dynamic, whose location in code is as yet undefined.
-    DynamicIndex(usize),
-    Index(usize),
-    Undefined,
+#[derive(BitfieldSpecifier, Copy, Clone, Debug, PartialEq)]
+#[bits = 7]
+pub enum IndexPtrTag {
+    DynamicUndefined = 0b1000101, // a predicate, declared as dynamic, whose location in code is as yet undefined.
+    DynamicIndex = 0b1000110,
+    Index = 0b1000111,
+    Undefined = 0b1001000,
 }
 
-#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
-pub struct CodeIndex(pub(crate) Rc<Cell<IndexPtr>>);
+#[bitfield]
+#[derive(Debug, Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)]
+pub struct IndexPtr {
+    pub p: B56,
+    #[allow(unused)] m: bool,
+    pub tag: IndexPtrTag,
+}
 
-impl Deref for CodeIndex {
-    type Target = Cell<IndexPtr>;
+impl IndexPtr {
+    pub(crate) fn dynamic_undefined() -> Self {
+        IndexPtr::new()
+            .with_p(0)
+            .with_m(false)
+            .with_tag(IndexPtrTag::DynamicUndefined)
+    }
 
-    #[inline]
-    fn deref(&self) -> &Self::Target {
-        self.0.deref()
+    pub(crate) fn undefined() -> Self {
+        IndexPtr::new()
+            .with_p(0)
+            .with_m(false)
+            .with_tag(IndexPtrTag::Undefined)
+    }
+
+    pub(crate) fn dynamic_index(p: usize) -> Self {
+        IndexPtr::new()
+            .with_p(p as u64)
+            .with_m(false)
+            .with_tag(IndexPtrTag::DynamicIndex)
+    }
+
+    pub(crate) fn index(p: usize) -> Self {
+        IndexPtr::new()
+            .with_p(p as u64)
+            .with_m(false)
+            .with_tag(IndexPtrTag::Index)
+    }
+}
+
+#[derive(Debug, Clone, Copy, Ord, Hash, PartialOrd, Eq, PartialEq)]
+pub struct CodeIndex(TypedArenaPtr<IndexPtr>);
+
+const_assert!(std::mem::align_of::<CodeIndex>() == 8);
+
+impl From<CodeIndex> for UntypedArenaPtr {
+    #[inline(always)]
+    fn from(ptr: CodeIndex) -> UntypedArenaPtr {
+        unsafe { std::mem::transmute(ptr.0.as_ptr()) }
+    }
+}
+
+impl From<UntypedArenaPtr> for CodeIndex {
+    #[inline(always)]
+    fn from(ptr: UntypedArenaPtr) -> CodeIndex {
+        CodeIndex(TypedArenaPtr::new(ptr.get_ptr() as *mut IndexPtr))
+    }
+}
+
+impl From<TypedArenaPtr<IndexPtr>> for CodeIndex {
+    #[inline(always)]
+    fn from(ptr: TypedArenaPtr<IndexPtr>) -> CodeIndex {
+        CodeIndex(ptr)
     }
 }
 
 impl CodeIndex {
     #[inline]
-    pub(super) fn new(ptr: IndexPtr) -> Self {
-        CodeIndex(Rc::new(Cell::new(ptr)))
+    pub(crate) fn new(ptr: IndexPtr, arena: &mut Arena) -> Self {
+        CodeIndex(arena_alloc!(ptr, arena))
     }
 
-    #[inline]
+    #[inline(always)]
+    pub(crate) fn default(arena: &mut Arena) -> Self {
+        CodeIndex::new(IndexPtr::undefined(), arena)
+    }
+
+    #[inline(always)]
     pub(crate) fn is_undefined(&self) -> bool {
-        match self.0.get() {
-            IndexPtr::Undefined => true, // | &IndexPtr::DynamicUndefined => true,
+        match self.0.tag() {
+            IndexPtrTag::Undefined => true, // | &IndexPtr::DynamicUndefined => true,
+            _ => false,
+        }
+    }
+
+
+    #[inline(always)]
+    pub(crate) fn is_dynamic_undefined(&self) -> bool {
+        match self.0.tag() {
+            IndexPtrTag::DynamicUndefined => true,
             _ => false,
         }
     }
 
     pub(crate) fn local(&self) -> Option<usize> {
-        match self.0.get() {
-            IndexPtr::Index(i) => Some(i),
-            IndexPtr::DynamicIndex(i) => Some(i),
+        match self.0.tag() {
+            IndexPtrTag::Index => Some(self.0.p() as usize),
+            IndexPtrTag::DynamicIndex => Some(self.0.p() as usize),
             _ => None,
         }
     }
-}
 
-impl Default for CodeIndex {
-    fn default() -> Self {
-        CodeIndex(Rc::new(Cell::new(IndexPtr::Undefined)))
+    #[inline(always)]
+    pub(crate) fn get(&self) -> IndexPtr {
+        *self.0.deref()
+    }
+
+    #[inline(always)]
+    pub(crate) fn set(&mut self, value: IndexPtr) {
+        *self.0.deref_mut() = value;
+    }
+
+    #[inline(always)]
+    pub(crate) fn replace(&mut self, value: IndexPtr) -> IndexPtr {
+        std::mem::replace(self.0.deref_mut(), value)
+    }
+
+    #[inline(always)]
+    pub(crate) fn as_ptr(&self) -> *const IndexPtr {
+        self.0.as_ptr()
     }
 }
 
@@ -269,19 +347,22 @@ impl IndexStore {
         module: Atom,
     ) -> Option<CodeIndex> {
         if module == atom!("user") {
-            match ClauseType::from(name, arity) {
-                ClauseType::Named(arity, name, _) => self.code_dir.get(&(name, arity)).cloned(),
-                _ => None,
-            }
+            /*match ClauseType::from(name, arity) {
+                ClauseType::Named(arity, name, _) => */
+            self.code_dir.get(&(name, arity)).cloned()
+            /*    _ => None,
+            }*/
         } else {
             self.modules
                 .get(&module)
-                .and_then(|module| match ClauseType::from(name, arity) {
-                    ClauseType::Named(arity, name, _) => {
+                .and_then(|module|/* |module| match ClauseType::from(name, arity) {
+                    ClauseType::Named(arity, name, _) => { */
                         module.code_dir.get(&(name, arity)).cloned()
+                    /*
                     }
                     _ => None,
-                })
+                } */
+                )
         }
     }
 

src/machine/machine_state.rs

@@ -418,6 +418,17 @@ impl MachineState {
                         }
                     }
                 }
+                (HeapCellValueTag::Str, s) => {
+                    let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                        .get_name_and_arity();
+
+                    if arity == 0 {
+                        if let Some(c) = name.as_char() {
+                            chars.push(c);
+                            continue;
+                        }
+                    }
+                }
                 _ => {
                 }
             );
@@ -454,6 +465,20 @@ impl MachineState {
         self.b0 = self.b;
     }
 
+    #[inline(always)]
+    pub fn neck_cut(&mut self) {
+        let b = self.b;
+        let b0 = self.b0;
+
+        if b > b0 {
+            self.b = b0;
+
+            if b > self.e {
+                self.stack.truncate(b);
+            }
+        }
+    }
+
     // Safety: the atom_tbl lives for the lifetime of the machine, as does the helper, so the ptr
     // will always be valid.
     pub fn read_term_from_user_input(&mut self, stream: Stream, indices: &mut IndexStore) -> CallResult {
@@ -666,6 +691,13 @@ impl MachineState {
                                         debug_assert_eq!(_arity, 0);
                                         var_names.insert(var, Rc::new(name.as_str().to_owned()));
                                     }
+                                    (HeapCellValueTag::Str, s) => {
+                                        let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                                            .get_name_and_arity();
+
+                                        debug_assert_eq!(arity, 0);
+                                        var_names.insert(var, Rc::new(name.as_str().to_owned()));
+                                    }
                                     _ => {
                                         unreachable!();
                                     }
@@ -677,34 +709,64 @@ impl MachineState {
                     );
                 }
 
+                let ignore_ops = read_heap_cell!(ignore_ops,
+                    (HeapCellValueTag::Atom, (name, _arity)) => {
+                        name == atom!("true")
+                    }
+                    (HeapCellValueTag::Str, s) => {
+                        let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                            .get_name_and_arity();
+
+                        debug_assert_eq!(arity, 0);
+                        name == atom!("true")
+                    }
+                    _ => {
+                        unreachable!()
+                    }
+                );
+
+                let numbervars = read_heap_cell!(numbervars,
+                    (HeapCellValueTag::Atom, (name, _arity)) => {
+                        name == atom!("true")
+                    }
+                    (HeapCellValueTag::Str, s) => {
+                        let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                            .get_name_and_arity();
+
+                        debug_assert_eq!(arity, 0);
+                        name == atom!("true")
+                    }
+                    _ => {
+                        unreachable!()
+                    }
+                );
+
+                let quoted = read_heap_cell!(quoted,
+                    (HeapCellValueTag::Atom, (name, _arity)) => {
+                        name == atom!("true")
+                    }
+                    (HeapCellValueTag::Str, s) => {
+                        let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                            .get_name_and_arity();
+
+                        debug_assert_eq!(arity, 0);
+                        name == atom!("true")
+                    }
+                    _ => {
+                        unreachable!()
+                    }
+                );
+
                 let mut printer = HCPrinter::new(
                     &mut self.heap,
-                    &mut self.arena,
                     op_dir,
                     PrinterOutputter::new(),
                     term_to_be_printed,
                 );
 
-                if let HeapCellValueTag::Atom = ignore_ops.get_tag() {
-                    let name = cell_as_atom!(ignore_ops);
-                    printer.ignore_ops = name == atom!("true");
-                } else {
-                    unreachable!();
-                }
-
-                if let HeapCellValueTag::Atom = numbervars.get_tag() {
-                    let name = cell_as_atom!(numbervars);
-                    printer.numbervars = name == atom!("true");
-                } else {
-                    unreachable!();
-                }
-
-                if let HeapCellValueTag::Atom = quoted.get_tag() {
-                    let name = cell_as_atom!(quoted);
-                    printer.quoted = name == atom!("true");
-                } else {
-                    unreachable!();
-                }
+                printer.ignore_ops = ignore_ops;
+                printer.numbervars = numbervars;
+                printer.quoted = quoted;
 
                 match Number::try_from(max_depth) {
                     Ok(Number::Fixnum(n)) => {

src/machine/machine_state_impl.rs

@@ -347,10 +347,27 @@ impl MachineState {
                 debug_assert_eq!(arity, 0);
                 self.fail = cstr_atom != atom!("[]");
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                if arity == 0 {
+                    self.fail = atom == atom!("") && name != atom!("[]");
+                } else {
+                    // this is intentionally the same policy for
+                    // value.tag() == Lis and PStrLoc. they're not
+                    // grouped together to allow for arity == 0.
+                    self.unify_partial_string(atom_as_cstr_cell!(atom), value);
+
+                    if !self.pdl.is_empty() {
+                        self.unify();
+                    }
+                }
+            }
             (HeapCellValueTag::CStr, cstr_atom) => {
                 self.fail = atom != cstr_atom;
             }
-            (HeapCellValueTag::Str | HeapCellValueTag::Lis | HeapCellValueTag::PStrLoc) => {
+            (HeapCellValueTag::Lis | HeapCellValueTag::PStrLoc) => {
                 self.unify_partial_string(atom_as_cstr_cell!(atom), value);
 
                 if !self.pdl.is_empty() {
@@ -553,6 +570,12 @@ impl MachineState {
             (HeapCellValueTag::Atom, (name, arity)) => {
                 self.fail = !(arity == 0 && name == atom);
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                self.fail = !(arity == 0 && name == atom);
+            }
             (HeapCellValueTag::CStr, cstr_atom) if atom == atom!("[]") => {
                 self.fail = cstr_atom != atom!("");
             }
@@ -587,6 +610,16 @@ impl MachineState {
                     self.fail = true;
                 }
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                if let Some(c2) = name.as_char() {
+                    self.fail = !(c == c2 && arity == 0);
+                } else {
+                    self.fail = true;
+                }
+            }
             (HeapCellValueTag::Char, c2) => {
                 if c != c2 {
                     self.fail = true;
@@ -739,11 +772,6 @@ impl MachineState {
                         self.unify_atom(name, d2);
                     }
                     (HeapCellValueTag::Str, s1) => {
-                        if d2.is_constant() {
-                            self.fail = true;
-                            break;
-                        }
-
                         if tabu_list.contains(&(d1, d2)) {
                             continue;
                         }
@@ -969,9 +997,7 @@ impl MachineState {
                 }
             }
             (HeapCellValueTag::Atom, (n2, a2)) => {
-                if !(a1 == 0 && a2 == 0 && n1 == n2) {
-                    self.fail = true;
-                }
+                self.fail = !(a1 == 0 && a2 == 0 && n1 == n2);
             }
             (HeapCellValueTag::AttrVar, h) => {
                 if self.bind_with_occurs_check(Ref::attr_var(h), str_loc_as_cell!(s1)) {
@@ -1285,11 +1311,6 @@ impl MachineState {
                         self.unify_atom(name, d2);
                     }
                     (HeapCellValueTag::Str, s1) => {
-                        if d2.is_constant() {
-                            self.fail = true;
-                            break;
-                        }
-
                         if tabu_list.contains(&(d1, d2)) {
                             continue;
                         }
@@ -1492,8 +1513,8 @@ impl MachineState {
             let v1 = self.store(s1);
             let v2 = self.store(s2);
 
-            let order_cat_v1 = v1.order_category();
-            let order_cat_v2 = v2.order_category();
+            let order_cat_v1 = v1.order_category(&self.heap);
+            let order_cat_v2 = v2.order_category(&self.heap);
 
             if order_cat_v1 != order_cat_v2 {
                 self.pdl.clear();
@@ -1552,6 +1573,15 @@ impl MachineState {
                                         );
                                     }
                                 }
+                                (HeapCellValueTag::Str, s) => {
+                                    let n2 = cell_as_atom_cell!(self.heap[s])
+                                        .get_name();
+
+                                    if n1 != n2 {
+                                        self.pdl.clear();
+                                        return Some(n1.cmp(&n2));
+                                    }
+                                }
                                 _ => {
                                     unreachable!();
                                 }
@@ -1579,11 +1609,67 @@ impl MachineState {
                                         return Some(c1.cmp(&c2));
                                     }
                                 }
+                                (HeapCellValueTag::Str, s) => {
+                                    let n2 = cell_as_atom_cell!(self.heap[s])
+                                        .get_name();
+
+                                    if let Some(c2) = n2.as_char() {
+                                        if c1 != c2 {
+                                            self.pdl.clear();
+                                            return Some(c1.cmp(&c2));
+                                        }
+                                    } else {
+                                        self.pdl.clear();
+                                        return Some(
+                                            Some(c1).cmp(&n2.chars().next())
+                                                    .then(Ordering::Less)
+                                        );
+                                    }
+                                }
                                 _ => {
                                     unreachable!()
                                 }
                             )
                         }
+                        (HeapCellValueTag::Str, s) => {
+                            let n1 = cell_as_atom_cell!(self.heap[s])
+                                .get_name();
+
+                            read_heap_cell!(v2,
+                                (HeapCellValueTag::Atom, (n2, _a2)) => {
+                                    if n1 != n2 {
+                                        self.pdl.clear();
+                                        return Some(n1.cmp(&n2));
+                                    }
+                                }
+                                (HeapCellValueTag::Char, c2) => {
+                                    if let Some(c1) = n1.as_char() {
+                                        if c1 != c2 {
+                                            self.pdl.clear();
+                                            return Some(c1.cmp(&c2));
+                                        }
+                                    } else {
+                                        self.pdl.clear();
+                                        return Some(
+                                            n1.chars().next().cmp(&Some(c2))
+                                              .then(Ordering::Greater)
+                                        );
+                                    }
+                                }
+                                (HeapCellValueTag::Str, s) => {
+                                    let n2 = cell_as_atom_cell!(self.heap[s])
+                                        .get_name();
+
+                                    if n1 != n2 {
+                                        self.pdl.clear();
+                                        return Some(n1.cmp(&n2));
+                                    }
+                                }
+                                _ => {
+                                    unreachable!();
+                                }
+                            )
+                        }
                         _ => {
                             unreachable!()
                         }
@@ -1597,8 +1683,8 @@ impl MachineState {
                     ) -> Option<Ordering> {
                         let compound = Some(TermOrderCategory::Compound);
 
-                        if iter2.focus.order_category() != compound {
-                            Some(compound.cmp(&iter2.focus.order_category()))
+                        if iter2.focus.order_category(iter2.heap) != compound {
+                            Some(compound.cmp(&iter2.focus.order_category(iter2.heap)))
                         } else {
                             let c1 = match iteratee {
                                 PStrIteratee::Char(_, c) => c,
@@ -2116,7 +2202,7 @@ impl MachineState {
                     heap_bound_deref(iter.heap, value),
                 ));
 
-                if value.is_compound() {
+                if value.is_compound(iter.heap) {
                     return true;
                 }
             }
@@ -2403,6 +2489,21 @@ impl MachineState {
                             a1.as_var().unwrap(),
                         );
                     }
+                    (HeapCellValueTag::Str, s) => {
+                        let (name, atom_arity) = cell_as_atom_cell!(self.heap[s])
+                            .get_name_and_arity();
+
+                        if atom_arity == 0 {
+                            self.try_functor_fabricate_struct(
+                                name,
+                                arity as usize,
+                                a1.as_var().unwrap(),
+                            );
+                        } else {
+                            let err = self.type_error(ValidType::Atomic, store_name);
+                            return Err(self.error_form(err, stub_gen()));
+                        }
+                    }
                     (HeapCellValueTag::Char, c) => {
                         let c = self.atom_tbl.build_with(&c.to_string());
 
@@ -2444,6 +2545,17 @@ impl MachineState {
                 let err = self.instantiation_error();
                 Err(self.error_form(err, stub_gen()))
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                if name == atom!("[]") && arity == 0 {
+                    Ok(vec![])
+                } else {
+                    let err = self.type_error(ValidType::List, value);
+                    Err(self.error_form(err, stub_gen()))
+                }
+            }
             (HeapCellValueTag::Atom, (name, arity)) => {
                 if name == atom!("[]") && arity == 0 {
                     Ok(vec![])
@@ -2484,6 +2596,17 @@ impl MachineState {
                 (HeapCellValueTag::PStrLoc, l) => {
                     return self.try_from_partial_string(result, l, stub_gen, a1);
                 }
+                (HeapCellValueTag::Str, s) => {
+                    let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                        .get_name_and_arity();
+
+                    if name == atom!("[]") && arity == 0 {
+                        break;
+                    } else {
+                        let err = self.type_error(ValidType::List, a1);
+                        return Err(self.error_form(err, stub_gen()));
+                    }
+                }
                 (HeapCellValueTag::Atom, (name, arity)) => {
                     if name == atom!("[]") && arity == 0 {
                         break;

src/machine/mock_wam.rs

@@ -61,7 +61,6 @@ impl MockWAM {
 
         let mut printer = HCPrinter::new(
             &mut self.machine_st.heap,
-            &mut self.machine_st.arena,
             &self.op_dir,
             PrinterOutputter::new(),
             heap_loc_as_cell!(term_write_result.heap_loc),
@@ -272,6 +271,7 @@ impl Machine {
 
         if let Some(ref mut builtins) = wam.indices.modules.get_mut(&atom!("builtins")) {
             load_module(
+                &mut wam.machine_st,
                 &mut wam.indices.code_dir,
                 &mut wam.indices.op_dir,
                 &mut wam.indices.meta_predicates,
@@ -297,6 +297,7 @@ impl Machine {
 
         if let Some(loader) = wam.indices.modules.get(&atom!("loader")) {
             load_module(
+                &mut wam.machine_st,
                 &mut wam.indices.code_dir,
                 &mut wam.indices.op_dir,
                 &mut wam.indices.meta_predicates,

src/machine/mod.rs

@@ -22,6 +22,7 @@ pub mod streams;
 pub mod system_calls;
 pub mod term_stream;
 
+use crate::arena::*;
 use crate::arithmetic::*;
 use crate::atom_table::*;
 use crate::forms::*;
@@ -160,6 +161,24 @@ pub(crate) fn import_builtin_impls(code_dir: &CodeDir, builtins: &mut Module) {
     }
 }
 
+#[inline]
+pub(crate) fn get_structure_index(value: HeapCellValue) -> Option<CodeIndex> {
+    read_heap_cell!(value,
+        (HeapCellValueTag::Cons, cons_ptr) => {
+             match_untyped_arena_ptr!(cons_ptr,
+                 (ArenaHeaderTag::IndexPtr, ip) => {
+                     return Some(CodeIndex::from(ip));
+                 }
+                 _ => {}
+             );
+        }
+        _ => {
+        }
+    );
+
+    None
+}
+
 impl Machine {
     #[inline]
     pub fn prelude_view_and_machine_st(&mut self) -> (MachinePreludeView, &mut MachineState) {
@@ -220,6 +239,7 @@ impl Machine {
 
         if let Some(toplevel) = self.indices.modules.get(&atom!("$toplevel")) {
             load_module(
+                &mut self.machine_st,
                 &mut self.indices.code_dir,
                 &mut self.indices.op_dir,
                 &mut self.indices.meta_predicates,
@@ -287,7 +307,7 @@ impl Machine {
     }
 
     pub(crate) fn configure_modules(&mut self) {
-        fn update_call_n_indices(loader: &Module, target_code_dir: &mut CodeDir) {
+        fn update_call_n_indices(loader: &Module, target_code_dir: &mut CodeDir, arena: &mut Arena) {
             for arity in 1..66 {
                 let key = (atom!("call"), arity);
 
@@ -295,7 +315,7 @@ impl Machine {
                     Some(src_code_index) => {
                         let target_code_index = target_code_dir
                             .entry(key)
-                            .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined));
+                            .or_insert_with(|| CodeIndex::new(IndexPtr::undefined(), arena));
 
                         target_code_index.set(src_code_index.get());
                     }
@@ -311,6 +331,7 @@ impl Machine {
                 // Import loader's exports into the builtins module so they will be
                 // implicitly included in every further module.
                 load_module(
+                    &mut self.machine_st,
                     &mut builtins.code_dir,
                     &mut builtins.op_dir,
                     &mut builtins.meta_predicates,
@@ -331,10 +352,10 @@ impl Machine {
             }
 
             for (_, target_module) in self.indices.modules.iter_mut() {
-                update_call_n_indices(&loader, &mut target_module.code_dir);
+                update_call_n_indices(&loader, &mut target_module.code_dir, &mut self.machine_st.arena);
             }
 
-            update_call_n_indices(&loader, &mut self.indices.code_dir);
+            update_call_n_indices(&loader, &mut self.indices.code_dir, &mut self.machine_st.arena);
 
             self.indices.modules.insert(atom!("loader"), loader);
         } else {
@@ -393,7 +414,10 @@ impl Machine {
 
         for (p, instr) in self.code[impls_offset ..].iter().enumerate() {
             let key = instr.to_name_and_arity();
-            self.indices.code_dir.insert(key, CodeIndex::new(IndexPtr::Index(p + impls_offset)));
+            self.indices.code_dir.insert(
+                key,
+                CodeIndex::new(IndexPtr::index(p + impls_offset), &mut self.machine_st.arena),
+            );
         }
     }
 
@@ -455,6 +479,7 @@ impl Machine {
 
         if let Some(builtins) = wam.indices.modules.get_mut(&atom!("builtins")) {
             load_module(
+                &mut wam.machine_st,
                 &mut wam.indices.code_dir,
                 &mut wam.indices.op_dir,
                 &mut wam.indices.meta_predicates,
@@ -480,6 +505,7 @@ impl Machine {
 
         if let Some(loader) = wam.indices.modules.get(&atom!("loader")) {
             load_module(
+                &mut wam.machine_st,
                 &mut wam.indices.code_dir,
                 &mut wam.indices.op_dir,
                 &mut wam.indices.meta_predicates,
@@ -666,18 +692,20 @@ impl Machine {
 
     #[inline(always)]
     fn try_call(&mut self, name: Atom, arity: usize, idx: IndexPtr) -> CallResult {
-        match idx {
-            IndexPtr::DynamicUndefined => {
+        let compiled_tl_index = idx.p() as usize;
+
+        match idx.tag() {
+            IndexPtrTag::DynamicUndefined => {
                 self.machine_st.fail = true;
             }
-            IndexPtr::Undefined => {
+            IndexPtrTag::Undefined => {
                 return Err(self.machine_st.throw_undefined_error(name, arity));
             }
-            IndexPtr::DynamicIndex(compiled_tl_index) => {
+            IndexPtrTag::DynamicIndex => {
                 self.machine_st.dynamic_mode = FirstOrNext::First;
                 self.machine_st.call_at_index(arity, compiled_tl_index);
             }
-            IndexPtr::Index(compiled_tl_index) => {
+            IndexPtrTag::Index => {
                 self.machine_st.call_at_index(arity, compiled_tl_index);
             }
         }
@@ -687,18 +715,20 @@ impl Machine {
 
     #[inline(always)]
     fn try_execute(&mut self, name: Atom, arity: usize, idx: IndexPtr) -> CallResult {
-        match idx {
-            IndexPtr::DynamicUndefined => {
+        let compiled_tl_index = idx.p() as usize;
+
+        match idx.tag() {
+            IndexPtrTag::DynamicUndefined => {
                 self.machine_st.fail = true;
             }
-            IndexPtr::Undefined => {
+            IndexPtrTag::Undefined => {
                 return Err(self.machine_st.throw_undefined_error(name, arity));
             }
-            IndexPtr::DynamicIndex(compiled_tl_index) => {
+            IndexPtrTag::DynamicIndex => {
                 self.machine_st.dynamic_mode = FirstOrNext::First;
                 self.machine_st.execute_at_index(arity, compiled_tl_index);
             }
-            IndexPtr::Index(compiled_tl_index) => {
+            IndexPtrTag::Index => {
                 self.machine_st.execute_at_index(arity, compiled_tl_index)
             }
         }

src/machine/preprocessor.rs

@@ -470,14 +470,133 @@ pub(super) fn setup_declaration<'a, LS: LoadState<'a>>(
     }
 }
 
+fn build_meta_predicate_clause<'a, LS: LoadState<'a>>(
+    loader: &mut Loader<'a, LS>,
+    module_name: Atom,
+    terms: Vec<Term>,
+    meta_specs: Vec<MetaSpec>,
+) -> Vec<Term> {
+    let mut arg_terms = Vec::with_capacity(terms.len());
+
+    for (term, meta_spec) in terms.into_iter().zip(meta_specs.iter()) {
+        if let MetaSpec::RequiresExpansionWithArgument(supp_args) = meta_spec {
+            if let Some(name) = term.name() {
+                if name == atom!("$call") {
+                    arg_terms.push(term);
+                    continue;
+                }
+
+                let arity = term.arity();
+
+                if let Term::Clause(_, _, ref terms) = &term {
+                    if let Some(Term::Literal(_, Literal::CodeIndex(_))) = terms.last() {
+                        arg_terms.push(term);
+                        continue;
+                    }
+                }
+
+                fn get_qualified_name(
+                    module_term: &Term,
+                    qualified_term: &Term,
+                ) -> Option<(Atom, Atom)> {
+                    if let Term::Literal(_, Literal::Atom(module_name)) = module_term {
+                        if let Some(name) = qualified_term.name() {
+                            return Some((*module_name, name));
+                        }
+                    }
+
+                    None
+                }
+
+                let (idx, term) = match term {
+                    Term::Clause(cell, atom!(":"), mut terms) if terms.len() == 2 => {
+                        if let Some((module_name, name)) = get_qualified_name(&terms[0], &terms[1])
+                        {
+                            (
+                                loader.get_or_insert_qualified_code_index(
+                                    module_name,
+                                    (name, terms[1].arity() + supp_args),
+                                ),
+                                terms.pop().unwrap(),
+                            )
+                        } else {
+                            arg_terms.push(Term::Clause(cell, atom!(":"), terms));
+                            continue;
+                        }
+                    }
+                    term => {
+                        (
+                            loader.get_or_insert_qualified_code_index(
+                                module_name,
+                                (name, arity + supp_args),
+                            ),
+                            term,
+                        )
+                    }
+                };
+
+                let term = match term {
+                    Term::Clause(cell, name, mut terms) => {
+                        terms.push(Term::Literal(Cell::default(), Literal::CodeIndex(idx)));
+                        Term::Clause(cell, name, terms)
+                    }
+                    Term::Literal(cell, Literal::Atom(name)) => {
+                        Term::Clause(
+                            cell,
+                            name,
+                            vec![Term::Literal(Cell::default(), Literal::CodeIndex(idx))],
+                        )
+                    }
+                    term => term,
+                };
+
+                arg_terms.push(term);
+                continue;
+            }
+        }
+
+        arg_terms.push(term);
+    }
+
+    arg_terms
+}
+
 #[inline]
 fn clause_to_query_term<'a, LS: LoadState<'a>>(
     loader: &mut Loader<'a, LS>,
     name: Atom,
-    terms: Vec<Term>,
+    mut terms: Vec<Term>,
     call_policy: CallPolicy,
 ) -> QueryTerm {
-    let ct = loader.get_clause_type(name, terms.len());
+    if let Some(Term::Literal(_, Literal::CodeIndex(_))) = terms.last() {
+        // supplementary code vector indices are unnecessary for
+        // root-level clauses.
+        terms.pop();
+    }
+
+    let mut ct = loader.get_clause_type(name, terms.len());
+
+    if let ClauseType::Named(arity, name, idx) = ct {
+        if let Some(meta_specs) = loader.get_meta_specs(name, arity).cloned() {
+            let module_name = loader.payload.compilation_target.module_name();
+            let terms = build_meta_predicate_clause(
+                loader,
+                module_name,
+                terms,
+                meta_specs,
+            );
+
+            return QueryTerm::Clause(
+                Cell::default(),
+                ClauseType::Named(arity, name, idx),
+                terms,
+                call_policy,
+            );
+        }
+
+        ct = ClauseType::Named(arity, name, idx);
+    }
+
     QueryTerm::Clause(Cell::default(), ct, terms, call_policy)
 }
 
@@ -486,13 +605,120 @@ fn qualified_clause_to_query_term<'a, LS: LoadState<'a>>(
     loader: &mut Loader<'a, LS>,
     module_name: Atom,
     name: Atom,
-    terms: Vec<Term>,
+    mut terms: Vec<Term>,
     call_policy: CallPolicy,
 ) -> QueryTerm {
-    let ct = loader.get_qualified_clause_type(module_name, name, terms.len());
+    if let Some(Term::Literal(_, Literal::CodeIndex(_))) = terms.last() {
+        // supplementary code vector indices are unnecessary for
+        // root-level clauses.
+        terms.pop();
+    }
+
+    let mut ct = loader.get_qualified_clause_type(module_name, name, terms.len());
+
+    if let ClauseType::Named(arity, name, idx) = ct {
+        if let Some(meta_specs) = loader.get_meta_specs(name, arity).cloned() {
+            let terms = build_meta_predicate_clause(
+                loader,
+                module_name,
+                terms,
+                meta_specs,
+            );
+
+            return QueryTerm::Clause(
+                Cell::default(),
+                ClauseType::Named(arity, name, idx),
+                terms,
+                call_policy,
+            );
+        }
+
+        ct = ClauseType::Named(arity, name, idx);
+    }
+
     QueryTerm::Clause(Cell::default(), ct, terms, call_policy)
 }
 
+fn compute_head(term: &Term) -> Vec<Term> {
+    let mut vars = IndexSet::new();
+
+    for term in post_order_iter(term) {
+        if let TermRef::Var(_, _, v) = term {
+            vars.insert(v.clone());
+        }
+    }
+
+    vars.insert(Rc::new(String::from("!")));
+    vars.into_iter()
+        .map(|v| Term::Var(Cell::default(), v))
+        .collect()
+}
+
+pub(crate) fn build_rule_body(vars: &[Term], body_term: Term) -> Term {
+    let head_term = Term::Clause(Cell::default(), atom!(""), vars.iter().cloned().collect());
+    let rule = vec![head_term, body_term];
+
+    Term::Clause(Cell::default(), atom!(":-"), rule)
+}
+
+// the terms form the body of the rule. We create a head, by
+// gathering variables from the body of terms and recording them
+// in the head clause.
+fn build_rule(body_term: Term) -> (JumpStub, VecDeque<Term>) {
+    // collect the vars of body_term into a head, return the num_vars
+    // (the arity) as well.
+    let vars = compute_head(&body_term);
+    let rule = build_rule_body(&vars, body_term);
+
+    (vars, VecDeque::from(vec![rule]))
+}
+
+fn build_disjunct(body_term: Term) -> (JumpStub, VecDeque<Term>) {
+    let vars = compute_head(&body_term);
+    let results = unfold_by_str(body_term, atom!(";"))
+        .into_iter()
+        .map(|term| {
+            let mut subterms = unfold_by_str(term, atom!(","));
+            mark_cut_variables(&mut subterms);
+
+            check_for_internal_if_then(&mut subterms);
+
+            let term = subterms.pop().unwrap();
+            let clause = fold_by_str(subterms.into_iter(), term, atom!(","));
+
+            build_rule_body(&vars, clause)
+        })
+        .collect();
+
+    (vars, results)
+}
+
+fn build_if_then(prec: Term, conq: Term) -> (JumpStub, VecDeque<Term>) {
+    let mut prec_seq = unfold_by_str(prec, atom!(","));
+    let comma_sym = atom!(",");
+    let cut_sym = Literal::Atom(atom!("!"));
+
+    prec_seq.push(Term::Literal(Cell::default(), cut_sym));
+
+    mark_cut_variables_as(&mut prec_seq, atom!("blocked_!"));
+
+    let mut conq_seq = unfold_by_str(conq, atom!(","));
+
+    mark_cut_variables(&mut conq_seq);
+    prec_seq.extend(conq_seq.into_iter());
+
+    let back_term = prec_seq.pop().unwrap();
+    let front_term = prec_seq.pop().unwrap();
+
+    let body_term = Term::Clause(
+        Cell::default(),
+        comma_sym,
+        vec![front_term, back_term],
+    );
+
+    build_rule(fold_by_str(prec_seq.into_iter(), body_term, comma_sym))
+}
+
 #[derive(Debug)]
 pub(crate) struct Preprocessor {
     queue: VecDeque<VecDeque<Term>>,
@@ -514,86 +740,6 @@ impl Preprocessor {
         }
     }
 
-    fn compute_head(&self, term: &Term) -> Vec<Term> {
-        let mut vars = IndexSet::new();
-
-        for term in post_order_iter(term) {
-            if let TermRef::Var(_, _, v) = term {
-                vars.insert(v.clone());
-            }
-        }
-
-        vars.insert(Rc::new(String::from("!")));
-        vars.into_iter()
-            .map(|v| Term::Var(Cell::default(), v))
-            .collect()
-    }
-
-    fn fabricate_rule_body(&self, vars: &Vec<Term>, body_term: Term) -> Term {
-        let head_term = Term::Clause(Cell::default(), atom!(""), vars.clone());
-        let rule = vec![head_term, body_term];
-
-        Term::Clause(Cell::default(), atom!(":-"), rule)
-    }
-
-    // the terms form the body of the rule. We create a head, by
-    // gathering variables from the body of terms and recording them
-    // in the head clause.
-    fn fabricate_rule(&self, body_term: Term) -> (JumpStub, VecDeque<Term>) {
-        // collect the vars of body_term into a head, return the num_vars
-        // (the arity) as well.
-        let vars = self.compute_head(&body_term);
-        let rule = self.fabricate_rule_body(&vars, body_term);
-
-        (vars, VecDeque::from(vec![rule]))
-    }
-
-    fn fabricate_disjunct(&self, body_term: Term) -> (JumpStub, VecDeque<Term>) {
-        let vars = self.compute_head(&body_term);
-        let results = unfold_by_str(body_term, atom!(";"))
-            .into_iter()
-            .map(|term| {
-                let mut subterms = unfold_by_str(term, atom!(","));
-                mark_cut_variables(&mut subterms);
-
-                check_for_internal_if_then(&mut subterms);
-
-                let term = subterms.pop().unwrap();
-                let clause = fold_by_str(subterms.into_iter(), term, atom!(","));
-
-                self.fabricate_rule_body(&vars, clause)
-            })
-            .collect();
-
-        (vars, results)
-    }
-
-    fn fabricate_if_then(&self, prec: Term, conq: Term) -> (JumpStub, VecDeque<Term>) {
-        let mut prec_seq = unfold_by_str(prec, atom!(","));
-        let comma_sym = atom!(",");
-        let cut_sym = Literal::Atom(atom!("!"));
-
-        prec_seq.push(Term::Literal(Cell::default(), cut_sym));
-
-        mark_cut_variables_as(&mut prec_seq, atom!("blocked_!"));
-
-        let mut conq_seq = unfold_by_str(conq, atom!(","));
-
-        mark_cut_variables(&mut conq_seq);
-        prec_seq.extend(conq_seq.into_iter());
-
-        let back_term = prec_seq.pop().unwrap();
-        let front_term = prec_seq.pop().unwrap();
-
-        let body_term = Term::Clause(
-            Cell::default(),
-            comma_sym,
-            vec![front_term, back_term],
-        );
-
-        self.fabricate_rule(fold_by_str(prec_seq.into_iter(), body_term, comma_sym))
-    }
-
     fn to_query_term<'a, LS: LoadState<'a>>(
         &mut self,
         loader: &mut Loader<'a, LS>,
@@ -605,7 +751,9 @@ impl Preprocessor {
                     Ok(QueryTerm::BlockedCut)
                 } else {
                     Ok(clause_to_query_term(
-                        loader, name, vec![],
+                        loader,
+                        name,
+                        vec![],
                         self.settings.default_call_policy(),
                     ))
                 }
@@ -618,7 +766,7 @@ impl Preprocessor {
                 (atom!(";"), 2) => {
                     let term = Term::Clause(r, name, terms);
 
-                    let (stub, clauses) = self.fabricate_disjunct(term);
+                    let (stub, clauses) = build_disjunct(term);
                     self.queue.push_back(clauses);
 
                     Ok(QueryTerm::Jump(stub))
@@ -627,7 +775,7 @@ impl Preprocessor {
                     let conq = terms.pop().unwrap();
                     let prec = terms.pop().unwrap();
 
-                    let (stub, clauses) = self.fabricate_if_then(prec, conq);
+                    let (stub, clauses) = build_if_then(prec, conq);
                     self.queue.push_back(clauses);
 
                     Ok(QueryTerm::Jump(stub))
@@ -644,7 +792,7 @@ impl Preprocessor {
                     let terms = vec![prec, conq];
 
                     let term = Term::Clause(Cell::default(), atom!(";"), terms);
-                    let (stub, clauses) = self.fabricate_disjunct(term);
+                    let (stub, clauses) = build_disjunct(term);
 
                     debug_assert!(clauses.len() > 0);
                     self.queue.push_back(clauses);
@@ -689,8 +837,8 @@ impl Preprocessor {
 
                             Ok(clause_to_query_term(
                                 loader,
-                                name,
-                                terms,
+                                atom!("call"),
+                                vec![Term::Clause(r, name, terms)],
                                 self.settings.default_call_policy(),
                             ))
                         }

src/machine/streams.rs

@@ -1254,6 +1254,18 @@ impl MachineState {
                     None
                 }
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+
+                if name != atom!("[]") {
+                    Some(name)
+                } else {
+                    None
+                }
+            }
             _ => {
                 None
             }
@@ -1270,6 +1282,19 @@ impl MachineState {
                     _ => unreachable!(),
                 }
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+
+                match name  {
+                    atom!("eof_code") => EOFAction::EOFCode,
+                    atom!("error") => EOFAction::Error,
+                    atom!("reset") => EOFAction::Reset,
+                    _ => unreachable!(),
+                }
+            }
             _ => {
                 unreachable!()
             }
@@ -1280,6 +1305,13 @@ impl MachineState {
                 debug_assert_eq!(arity, 0);
                 name == atom!("true")
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+                name == atom!("true")
+            }
             _ => {
                 unreachable!()
             }
@@ -1294,6 +1326,17 @@ impl MachineState {
                     _ => unreachable!(),
                 }
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+                match name {
+                    atom!("text") => StreamType::Text,
+                    atom!("binary") => StreamType::Binary,
+                    _ => unreachable!(),
+                }
+            }
             _ => {
                 unreachable!()
             }
@@ -1334,6 +1377,24 @@ impl MachineState {
                     }
                 };
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+
+                return match stream_aliases.get(&name) {
+                    Some(stream) if !stream.is_null_stream() => Ok(*stream),
+                    _ => {
+                        let stub = functor_stub(caller, arity);
+                        let addr = atom_as_cell!(name);
+
+                        let existence_error = self.existence_error(ExistenceError::Stream(addr));
+
+                        Err(self.error_form(existence_error, stub))
+                    }
+                };
+            }
             (HeapCellValueTag::Cons, ptr) => {
                 match_untyped_arena_ptr!(ptr,
                      (ArenaHeaderTag::Stream, stream) => {
@@ -1402,9 +1463,9 @@ impl MachineState {
         arity: usize,
     ) -> MachineStub {
         let stub = functor_stub(caller, arity);
-        let err = self.permission_error(perm, err_atom, stream_as_cell!(stream));
+        let err  = self.permission_error(perm, err_atom, stream_as_cell!(stream));
 
-        return self.error_form(err, stub);
+        self.error_form(err, stub)
     }
 
     #[inline]
@@ -1430,9 +1491,9 @@ impl MachineState {
         stub_arity: usize,
     ) -> MachineStub {
         let stub = functor_stub(stub_name, stub_arity);
-        let err = self.permission_error(Permission::Open, atom!("source_sink"), culprit);
+        let err  = self.permission_error(Permission::Open, atom!("source_sink"), culprit);
 
-        return self.error_form(err, stub);
+        self.error_form(err, stub)
     }
 
     pub(crate) fn occupied_alias_permission_error(
@@ -1442,24 +1503,22 @@ impl MachineState {
         stub_arity: usize,
     ) -> MachineStub {
         let stub = functor_stub(stub_name, stub_arity);
-        let alias_name = atom!("alias");
 
         let err = self.permission_error(
             Permission::Open,
             atom!("source_sink"),
-            functor!(alias_name, [atom(alias)]),
+            functor!(atom!("alias"), [atom(alias)]),
         );
 
-        return self.error_form(err, stub);
+        self.error_form(err, stub)
     }
 
     pub(crate) fn reposition_error(&mut self, stub_name: Atom, stub_arity: usize) -> MachineStub {
         let stub = functor_stub(stub_name, stub_arity);
-
         let rep_stub = functor!(atom!("reposition"), [atom(atom!("true"))]);
         let err = self.permission_error(Permission::Open, atom!("source_sink"), rep_stub);
 
-        return self.error_form(err, stub);
+        self.error_form(err, stub)
     }
 
     pub(crate) fn check_stream_properties(

src/machine/system_calls.rs

@@ -10,7 +10,7 @@ use crate::heap_iter::*;
 use crate::heap_print::*;
 use crate::instructions::*;
 use crate::machine;
-use crate::machine::{Machine, VERIFY_ATTR_INTERRUPT_LOC};
+use crate::machine::{Machine, VERIFY_ATTR_INTERRUPT_LOC, get_structure_index};
 use crate::machine::code_walker::*;
 use crate::machine::copier::*;
 use crate::machine::heap::*;
@@ -29,22 +29,27 @@ use crate::types::*;
 
 use ordered_float::OrderedFloat;
 
+use fxhash::{FxBuildHasher, FxHasher};
 use indexmap::IndexSet;
 
 use ref_thread_local::{RefThreadLocal, ref_thread_local};
 
+use std::cell::Cell;
+use std::cmp::Ordering;
 use std::collections::BTreeSet;
 use std::convert::TryFrom;
 use std::env;
 use std::fs;
+use std::hash::{BuildHasher, BuildHasherDefault};
 use std::io::{ErrorKind, Read, Write};
 use std::iter::{once, FromIterator};
 use std::mem;
 use std::net::{TcpListener, TcpStream};
 use std::num::NonZeroU32;
 use std::ops::Sub;
-use std::str::FromStr;
 use std::process;
+use std::rc::Rc;
+use std::str::FromStr;
 
 use chrono::{offset::Local, DateTime};
 use cpu_time::ProcessTime;
@@ -173,6 +178,16 @@ impl BrentAlgState {
                     CycleSearchResult::NotList(self.num_steps(), heap[self.hare])
                 };
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(heap[s])
+                    .get_name_and_arity();
+
+                return if name == atom!("[]") && arity == 0 {
+                    CycleSearchResult::ProperList(self.num_steps())
+                } else {
+                    CycleSearchResult::NotList(self.num_steps(), heap[self.hare])
+                };
+            }
             (HeapCellValueTag::Lis, l) => {
                 return CycleSearchResult::UntouchedList(self.num_steps(), l);
             }
@@ -439,6 +454,30 @@ impl BrentAlgState {
 }
 
 impl MachineState {
+    #[inline]
+    pub(crate) fn variable_set<S: BuildHasher>(
+        &mut self,
+        seen_set: &mut IndexSet<HeapCellValue, S>,
+        value: HeapCellValue,
+    ) {
+        let mut iter = stackful_preorder_iter(&mut self.heap, value);
+
+        while let Some(value) = iter.next() {
+            let value = unmark_cell_bits!(value);
+
+            if value.is_var() {
+                let value = unmark_cell_bits!(heap_bound_store(
+                    iter.heap,
+                    heap_bound_deref(iter.heap, value)
+                ));
+
+                if value.is_var() {
+                    seen_set.insert(value);
+                }
+            }
+        }
+    }
+
     fn skip_max_list_cycle(&mut self, lam: usize) {
         fn step(heap: &[HeapCellValue], mut value: HeapCellValue) -> usize {
             loop {
@@ -820,6 +859,16 @@ impl MachineState {
                     None
                 }
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.heap[s])
+                    .get_name_and_arity();
+
+                if arity == 0 {
+                    Some(AtomOrString::Atom(name))
+                } else {
+                    None
+                }
+            }
             (HeapCellValueTag::Char, c) => {
                 Some(AtomOrString::String(c.to_string()))
             }
@@ -924,6 +973,289 @@ impl MachineState {
 }
 
 impl Machine {
+    #[inline(always)]
+    pub(crate) fn call_inline(
+        &mut self,
+        arity: usize,
+        call_at_index: impl Fn(&mut Machine, Atom, usize, IndexPtr) -> CallResult,
+    ) -> CallResult {
+        let arity = arity - 1;
+        let goal = self.machine_st.store(self.machine_st.deref(self.machine_st.registers[1]));
+
+        let load_registers = |machine_st: &mut MachineState, goal: HeapCellValue| -> Option<PredicateKey> {
+            read_heap_cell!(goal,
+                (HeapCellValueTag::Str, s) => {
+                    let (name, goal_arity) = cell_as_atom_cell!(machine_st.heap[s])
+                        .get_name_and_arity();
+
+                    if goal_arity > 0 {
+                        for idx in (1 .. arity + 1).rev() {
+                            machine_st.registers[idx + goal_arity] = machine_st.registers[idx + 1];
+                        }
+                    } else {
+                        for idx in 1 .. arity + 1 {
+                            machine_st.registers[idx] = machine_st.registers[idx + 1];
+                        }
+                    }
+
+                    for idx in 1 .. goal_arity + 1 {
+                        machine_st.registers[idx] = machine_st.heap[s+idx];
+                    }
+
+                    Some((name, goal_arity))
+                }
+                _ => {
+                    unreachable!()
+                }
+            )
+        };
+
+        read_heap_cell!(goal,
+            (HeapCellValueTag::Str, s) => {
+                let goal_arity = cell_as_atom_cell!(self.machine_st.heap[s]).get_arity();
+
+                if self.machine_st.heap.len() > s + goal_arity + 1 {
+                    let index_cell = self.machine_st.heap[s+goal_arity+1];
+
+                    if let Some(code_index) = get_structure_index(index_cell) {
+                        if code_index.is_undefined() || code_index.is_dynamic_undefined() {
+                            self.machine_st.fail = true;
+                            return Ok(());
+                        }
+
+                        match load_registers(&mut self.machine_st, goal) {
+                            Some((name, goal_arity)) => {
+                                let arity = goal_arity + arity;
+                                self.machine_st.neck_cut();
+                                return call_at_index(self, name, arity, code_index.get());
+                            }
+                            None => {
+                            }
+                        }
+                    }
+                }
+            }
+            _ => {
+            }
+        );
+
+        self.machine_st.fail = true;
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub(crate) fn compile_inline_or_expanded_goal(&mut self) -> CallResult {
+        let goal = self.machine_st.store(self.machine_st.deref(self.machine_st.registers[1]));
+        let module_name = self.machine_st.store(self.machine_st.deref(self.machine_st.registers[4]));
+
+        // supp_vars are the supplementary variables generated by
+        // complete_partial_goal prior to goal_expansion.
+        let mut supp_vars = IndexSet::with_hasher(FxBuildHasher::default());
+
+        self.machine_st.variable_set(&mut supp_vars, self.machine_st.registers[2]);
+
+        struct GoalAnalysisResult {
+            is_simple_goal: bool,
+            goal: HeapCellValue,
+            key: PredicateKey,
+            expanded_vars: IndexSet<HeapCellValue, BuildHasherDefault<FxHasher>>,
+            supp_vars: IndexSet<HeapCellValue, BuildHasherDefault<FxHasher>>,
+        }
+
+        let result = read_heap_cell!(goal,
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                let mut expanded_vars = IndexSet::with_hasher(FxBuildHasher::default());
+
+                // fill expanded_vars with variables of the partial
+                // goal pre-completion by complete_partial_goal.
+                for idx in s + 1 .. s + arity - supp_vars.len() + 1 {
+                    self.machine_st.variable_set(&mut expanded_vars, self.machine_st.heap[idx]);
+                }
+
+                let is_simple_goal = if arity >= supp_vars.len() {
+                    // post_supp_args are the arguments to the
+                    // post-expansion complete goal gathered from the
+                    // final supp_vars.len() arguments. they must
+                    // agree in supp_vars in order of entry of
+                    // insertion as well as the previous
+                    // supp_vars.len() argument's variables being
+                    // disjoint from them. if they are not, the
+                    // expanded goal are not simple.
+
+                    let post_supp_args = self.machine_st.heap[s+arity-supp_vars.len()+1 .. s+arity+1]
+                        .iter()
+                        .cloned();
+
+                    post_supp_args
+                        .zip(supp_vars.iter())
+                        .all(|(arg_term, supp_var)| {
+                            let arg_term = self.machine_st.store(self.machine_st.deref(arg_term));
+
+                            if arg_term.is_var() && supp_var.is_var() {
+                                return arg_term == *supp_var;
+                            }
+
+                            false
+                        }) && expanded_vars.intersection(&supp_vars).next().is_none()
+                } else {
+                    false
+                };
+
+                let goal = if is_simple_goal {
+                    let h = self.machine_st.heap.len();
+                    let arity = arity - supp_vars.len();
+
+                    for idx in 0 .. arity + 1 {
+                        let value = self.machine_st.heap[s + idx];
+                        self.machine_st.heap.push(value);
+                    }
+
+                    self.machine_st.heap[h] = atom_as_cell!(name, arity);
+
+                    str_loc_as_cell!(h)
+                } else {
+                    goal
+                };
+
+                GoalAnalysisResult {
+                    is_simple_goal,
+                    goal,
+                    key: (name, arity),
+                    expanded_vars,
+                    supp_vars
+                }
+            }
+            (HeapCellValueTag::Atom, (name, arity)) => {
+                debug_assert_eq!(arity, 0);
+
+                let h = self.machine_st.heap.len();
+                self.machine_st.heap.push(goal);
+
+                GoalAnalysisResult {
+                    is_simple_goal: true,
+                    goal: str_loc_as_cell!(h),
+                    key: (name, 0),
+                    expanded_vars: IndexSet::with_hasher(FxBuildHasher::default()),
+                    supp_vars,
+                }
+            }
+            (HeapCellValueTag::Char, c) => {
+                let name = self.machine_st.atom_tbl.build_with(&c.to_string());
+
+                let h = self.machine_st.heap.len();
+                self.machine_st.heap.push(atom_as_cell!(name));
+
+                GoalAnalysisResult {
+                    is_simple_goal: true,
+                    goal: str_loc_as_cell!(h),
+                    key: (name, 0),
+                    expanded_vars: IndexSet::with_hasher(FxBuildHasher::default()),
+                    supp_vars,
+                }
+            }
+            _ => {
+                self.machine_st.fail = true;
+                return Ok(());
+            }
+        );
+
+        if result.key.0 == atom!(":") {
+            self.machine_st.fail = true;
+            return Ok(());
+        }
+
+        let expanded_term = if result.is_simple_goal {
+            let idx = self.get_or_insert_qualified_code_index(module_name, result.key);
+            self.machine_st.heap.push(untyped_arena_ptr_as_cell!(UntypedArenaPtr::from(idx)));
+            result.goal
+        } else {
+            // all supp_vars must appear later!
+            let vars = IndexSet::<HeapCellValue, BuildHasherDefault<FxHasher>>::from_iter(
+                result.expanded_vars.difference(&result.supp_vars).cloned()
+            );
+
+            let vars: Vec<_> = vars
+                .union(&result.supp_vars) // difference + union does not cancel.
+                .map(|v| Term::Var(Cell::default(), Rc::new(format!("_{}", v.get_value()))))
+                .collect();
+
+            let helper_clause_loc = self.code.len();
+
+            match self.compile_standalone_clause(temp_v!(1), &vars) {
+                Err(e) => {
+                    let err = self.machine_st.session_error(e);
+                    let stub = functor_stub(atom!("call"), result.key.1);
+
+                    return Err(self.machine_st.error_form(err, stub));
+                }
+                Ok(()) => {
+                    let h = self.machine_st.heap.len();
+
+                    self.machine_st.heap.push(atom_as_cell!(atom!("$aux"), 0));
+
+                    for value in result.expanded_vars.difference(&result.supp_vars).cloned() {
+                        self.machine_st.heap.push(value);
+                    }
+
+                    let anon_str_arity = self.machine_st.heap.len() - h - 1;
+
+                    self.machine_st.heap[h] = atom_as_cell!(atom!("$aux"), anon_str_arity);
+
+                    let idx = CodeIndex::new(
+                        IndexPtr::index(helper_clause_loc),
+                        &mut self.machine_st.arena,
+                    );
+
+                    self.machine_st.heap.push(untyped_arena_ptr_as_cell!(UntypedArenaPtr::from(idx)));
+
+                    str_loc_as_cell!(h)
+                }
+            }
+        };
+
+        let truncated_goal = self.machine_st.registers[3];
+        unify!(&mut self.machine_st, expanded_term, truncated_goal);
+
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub(crate) fn is_expanded_or_inlined(&mut self) {
+        let (_module_loc, qualified_goal) = self.machine_st.strip_module(
+            self.machine_st.registers[1],
+            empty_list_as_cell!(),
+        );
+
+        if HeapCellValueTag::Str == qualified_goal.get_tag() {
+            let s = qualified_goal.get_value();
+            let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                .get_name_and_arity();
+
+            if name == atom!("$call") {
+                return;
+            }
+
+            if self.machine_st.heap.len() > s + 1 + arity {
+                let idx_cell = self.machine_st.heap[s + 1 + arity];
+
+                if HeapCellValueTag::Cons == idx_cell.get_tag() {
+                    match_untyped_arena_ptr!(cell_as_untyped_arena_ptr!(idx_cell),
+                        (ArenaHeaderTag::IndexPtr, _ip) => {
+                            return;
+                        }
+                        _ => {
+                        }
+                    );
+                }
+            }
+        }
+
+        self.machine_st.fail = true;
+    }
+
     #[inline(always)]
     pub(crate) fn prepare_call_clause(&mut self, arity: usize) -> CallResult {
         let (module_loc, qualified_goal) = self.machine_st.strip_module(
@@ -1391,6 +1723,19 @@ impl Machine {
 
                 unify!(self.machine_st, self.machine_st.registers[2], list_loc_as_cell!(h));
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                if arity == 0 {
+                    self.machine_st.unify_complete_string(
+                        name,
+                        self.machine_st.store(self.machine_st.deref(self.machine_st.registers[2])),
+                    );
+                } else {
+                    self.machine_st.fail = true;
+                }
+            }
             (HeapCellValueTag::Atom, (name, arity)) => {
                 if arity == 0 {
                     self.machine_st.unify_complete_string(
@@ -1454,6 +1799,20 @@ impl Machine {
                     self.machine_st.fail = true;
                 }
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                if arity == 0 {
+                    let iter = name.chars()
+                        .map(|c| fixnum_as_cell!(Fixnum::build_with(c as i64)));
+
+                    let h = iter_to_heap_list(&mut self.machine_st.heap, iter);
+                    unify!(self.machine_st, heap_loc_as_cell!(h), self.machine_st.registers[2]);
+                } else {
+                    self.machine_st.fail = true;
+                }
+            }
             (HeapCellValueTag::Var | HeapCellValueTag::AttrVar | HeapCellValueTag::StackVar) => {
                 let stub_gen = || functor_stub(atom!("atom_codes"), 2);
 
@@ -1482,6 +1841,17 @@ impl Machine {
         let a1 = self.machine_st.store(self.machine_st.deref(self.machine_st.registers[1]));
 
         let len: i64 = read_heap_cell!(a1,
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                if arity == 0 {
+                    name.chars().count() as i64
+                } else {
+                    self.machine_st.fail = true;
+                    return;
+                }
+            }
             (HeapCellValueTag::Atom, (name, arity)) => {
                 if arity == 0 {
                     name.chars().count() as i64
@@ -2019,6 +2389,13 @@ impl Machine {
             (HeapCellValueTag::Atom, (name, _arity)) => {
                 name.as_char().unwrap()
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+                name.as_char().unwrap()
+            }
             (HeapCellValueTag::Char, c) => {
                 c
             }
@@ -2080,6 +2457,13 @@ impl Machine {
             (HeapCellValueTag::Atom, (name, _arity)) => {
                 name.as_char().unwrap()
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+                name.as_char().unwrap()
+            }
             _ => {
                 unreachable!()
             }
@@ -2985,7 +3369,7 @@ impl Machine {
     }
 
     #[inline(always)]
-    pub(crate) fn delete_head_attribute(&mut self) {
+     pub(crate) fn delete_head_attribute(&mut self) {
         let addr = self.machine_st.store(self.machine_st.deref(self.machine_st.registers[1]));
 
         debug_assert_eq!(addr.get_tag(), HeapCellValueTag::AttrVar);
@@ -3016,18 +3400,48 @@ impl Machine {
         &mut self,
         narity: usize,
     ) -> Result<(Atom, PredicateKey), MachineStub> {
-        let module_name = cell_as_atom!(self.machine_st.store(self.machine_st.deref(
-            self.machine_st.registers[1 + narity]
-        )));
+        let module_name = self.machine_st.store(self.machine_st.deref(
+            self.machine_st.registers[1]
+        ));
+
+        let module_name = read_heap_cell!(module_name,
+            (HeapCellValueTag::Atom, (name, _arity)) => {
+                debug_assert_eq!(_arity, 0);
+                name
+            }
+            (HeapCellValueTag::Str, s) => {
+                let (module_name, _arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(_arity, 0);
+                module_name
+            }
+            _ if module_name.is_var() => {
+                atom!("user")
+            }
+            _ => {
+                unreachable!()
+            }
+        );
 
         let goal = self.machine_st.store(self.machine_st.deref(
-            self.machine_st.registers[2 + narity]
+            self.machine_st.registers[2]
         ));
 
         let (name, arity, s) = self.machine_st.setup_call_n_init_goal_info(goal, narity)?;
 
-        for i in (arity + 1..arity + narity + 1).rev() {
-            self.machine_st.registers[i] = self.machine_st.registers[i - arity];
+        match arity.cmp(&2) {
+            Ordering::Less => {
+                for i in arity + 1..arity + narity + 1 {
+                    self.machine_st.registers[i] = self.machine_st.registers[i + 2 - arity];
+                }
+            }
+            Ordering::Greater => {
+                for i in (arity + 1..arity + narity + 1).rev() {
+                    self.machine_st.registers[i] = self.machine_st.registers[i + 2 - arity];
+                }
+            }
+            Ordering::Equal => {}
         }
 
         for i in 1..arity + 1 {
@@ -3122,6 +3536,9 @@ impl Machine {
                     (HeapCellValueTag::Atom, (name, _arity)) => {
                         name
                     }
+                    (HeapCellValueTag::Str, s) => {
+                        cell_as_atom!(self.machine_st.heap[s])
+                    }
                     (HeapCellValueTag::Char, c) => {
                         self.machine_st.atom_tbl.build_with(&c.to_string())
                     }
@@ -3544,6 +3961,9 @@ impl Machine {
             (HeapCellValueTag::Atom, (name, _arity)) => {
                 name
             }
+            (HeapCellValueTag::Str, s) => {
+                cell_as_atom!(self.machine_st.heap[s])
+            }
             _ => {
                 unreachable!()
             }
@@ -3906,7 +4326,7 @@ impl Machine {
                 let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
                     .get_name_and_arity();
 
-                let ct = ClauseType::from(name, arity);
+                let ct = ClauseType::from(name, arity, &mut self.machine_st.arena);
 
                 if ct.is_inlined() || ct.is_builtin() {
                     true
@@ -3917,10 +4337,10 @@ impl Machine {
                         module_name,
                     )
                         .map(|index| index.get())
-                        .unwrap_or(IndexPtr::DynamicUndefined);
+                        .unwrap_or(IndexPtr::dynamic_undefined());
 
-                    match index {
-                        IndexPtr::DynamicUndefined | IndexPtr::Undefined => false,
+                    match index.tag() {
+                        IndexPtrTag::DynamicUndefined | IndexPtrTag::Undefined => false,
                         _ => true,
                     }
                 }
@@ -3928,7 +4348,7 @@ impl Machine {
             (HeapCellValueTag::Atom, (name, arity)) => {
                 debug_assert_eq!(arity, 0);
 
-                let ct = ClauseType::from(name, 0);
+                let ct = ClauseType::from(name, 0, &mut self.machine_st.arena);
 
                 if ct.is_inlined() || ct.is_builtin() {
                     true
@@ -3939,10 +4359,10 @@ impl Machine {
                         module_name,
                     )
                         .map(|index| index.get())
-                        .unwrap_or(IndexPtr::DynamicUndefined);
+                        .unwrap_or(IndexPtr::dynamic_undefined());
 
-                    match index {
-                        IndexPtr::DynamicUndefined => false,
+                    match index.tag() {
+                        IndexPtrTag::DynamicUndefined => false,
                         _ => true,
                     }
                 }
@@ -4208,7 +4628,9 @@ impl Machine {
             LOC_INIT.call_once(|| {
                 if let Some(builtins) = self.indices.modules.get(&atom!("builtins")) {
                     match builtins.code_dir.get(&(atom!("staggered_sc"), 2)).map(|cell| cell.get()) {
-                        Some(IndexPtr::Index(p)) => {
+                        Some(ip) if ip.tag() == IndexPtrTag::Index => {
+                            let p = ip.p() as usize;
+
                             match &self.code[p] {
                                 &Instruction::TryMeElse(o) => {
                                     SEMICOLON_SECOND_BRANCH_LOC = p + o;
@@ -4253,30 +4675,40 @@ impl Machine {
     pub(crate) fn next_ep(&mut self) {
         let first_arg = self.machine_st.store(self.machine_st.deref(self.machine_st.registers[1]));
 
+        let next_ep_atom = |machine_st: &mut MachineState, name, arity| {
+            debug_assert_eq!(name, atom!("first"));
+            debug_assert_eq!(arity, 0);
+
+            if machine_st.e == 0 {
+                machine_st.fail = true;
+                return;
+            }
+
+            let and_frame = machine_st.stack.index_and_frame(machine_st.e);
+            let cp = and_frame.prelude.cp - 1;
+
+            let e = and_frame.prelude.e;
+            let e = Fixnum::build_with(i64::try_from(e).unwrap());
+
+            let p = str_loc_as_cell!(machine_st.heap.len());
+
+            machine_st.heap.extend(functor!(atom!("dir_entry"), [fixnum(cp)]));
+            machine_st.unify_fixnum(e, machine_st.registers[2]);
+
+            if !machine_st.fail {
+                unify!(machine_st, p, machine_st.registers[3]);
+            }
+        };
+
         read_heap_cell!(first_arg,
             (HeapCellValueTag::Atom, (name, arity)) => {
-                debug_assert_eq!(name, atom!("first"));
-                debug_assert_eq!(arity, 0);
+                next_ep_atom(&mut self.machine_st, name, arity);
+            }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
 
-                if self.machine_st.e == 0 {
-                    self.machine_st.fail = true;
-                    return;
-                }
-
-                let and_frame = self.machine_st.stack.index_and_frame(self.machine_st.e);
-                let cp = and_frame.prelude.cp - 1;
-
-                let e = and_frame.prelude.e;
-                let e = Fixnum::build_with(i64::try_from(e).unwrap());
-
-                let p = str_loc_as_cell!(self.machine_st.heap.len());
-
-                self.machine_st.heap.extend(functor!(atom!("dir_entry"), [fixnum(cp)]));
-                self.machine_st.unify_fixnum(e, self.machine_st.registers[2]);
-
-                if !self.machine_st.fail {
-                    unify!(self.machine_st, p, self.machine_st.registers[3]);
-                }
+                next_ep_atom(&mut self.machine_st, name, arity);
             }
             (HeapCellValueTag::Fixnum, n) => {
                 let e = n.get_num() as usize;
@@ -4346,6 +4778,13 @@ impl Machine {
                 debug_assert_eq!(arity, 0);
                 self.machine_st.fail = non_quoted_token(name.as_str().chars());
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+                self.machine_st.fail = non_quoted_token(name.as_str().chars());
+            }
             _ => {
                 self.machine_st.fail = true;
             }
@@ -4480,6 +4919,13 @@ impl Machine {
                 debug_assert_eq!(arity, 0);
                 name
             }
+            (HeapCellValueTag::Str, s) => {
+                let (name, arity) = cell_as_atom_cell!(self.machine_st.heap[s])
+                    .get_name_and_arity();
+
+                debug_assert_eq!(arity, 0);
+                name
+            }
             _ => {
                 self.machine_st.atom_tbl.build_with(&match Number::try_from(port) {
                     Ok(Number::Fixnum(n)) => n.get_num().to_string(),
@@ -5011,26 +5457,9 @@ impl Machine {
             return;
         }
 
-        let mut seen_set = IndexSet::new();
+        let mut seen_set = IndexSet::with_hasher(FxBuildHasher::default());
 
-        {
-            let mut iter = stackful_preorder_iter(&mut self.machine_st.heap, stored_v);
-
-            while let Some(value) = iter.next() {
-                let value = unmark_cell_bits!(value);
-
-                if value.is_var() {
-                    let value = unmark_cell_bits!(heap_bound_store(
-                        iter.heap,
-                        heap_bound_deref(iter.heap, value)
-                    ));
-
-                    if value.is_var() {
-                        seen_set.insert(value);
-                    }
-                }
-            }
-        }
+        self.machine_st.variable_set(&mut seen_set, stored_v);
 
         let outcome = heap_loc_as_cell!(
             iter_to_heap_list(&mut self.machine_st.heap, seen_set.into_iter())
@@ -5271,9 +5700,15 @@ impl Machine {
         };
 
         let result = printer.print().result();
-        let chars = put_complete_string(&mut self.machine_st.heap, &result, &mut self.machine_st.atom_tbl);
+        let chars = put_complete_string(
+            &mut self.machine_st.heap,
+            &result,
+            &mut self.machine_st.atom_tbl,
+        );
 
-        let result_addr = self.machine_st.store(self.machine_st.deref(self.machine_st.registers[1]));
+        let result_addr = self.machine_st.store(self.machine_st.deref(
+            self.machine_st.registers[1]
+        ));
 
         if let Some(var) = result_addr.as_var() {
             self.machine_st.bind(var, chars);