move to a predicate-based module system, move to loader.[rs|pl]-based compilation, add support for incremental clause compilation

src/machine/attributed_variables.pl

@@ -13,10 +13,16 @@ iterate([Var|VarBindings], [Value|ValueBindings], [ListOfGoalLists | ListsCubed]
     iterate(VarBindings, ValueBindings, ListsCubed).
 iterate([], [], []).
 
+/*
 gather_modules(Attrs, []) :- var(Attrs), !.
 gather_modules([Attr|Attrs], [Module|Modules]) :-
     '$module_of'(Module, Attr),  % write the owning module of Attr to Module.
     gather_modules(Attrs, Modules).
+*/
+
+gather_modules(Attrs, []) :- var(Attrs), !.
+gather_modules([Module:_|Attrs], [Module|Modules]) :-
+    gather_modules(Attrs, Modules).
 
 call_verify_attributes(Attrs, _, _, []) :-
     var(Attrs), !.
@@ -24,7 +30,7 @@ call_verify_attributes([], _, _, []).
 call_verify_attributes([Attr|Attrs], Var, Value, ListOfGoalLists) :-
     gather_modules([Attr|Attrs], Modules0),
     sort(Modules0, Modules),
-    verify_attrs(Modules, Var, Value, ListOfGoalLists).
+    verify_attrs(Modules, Var, Value, ListOfGoalLists). % verify_attrs(Modules, Var, Value, ListOfGoalLists).
 
 verify_attrs([Module|Modules], Var, Value, [Goals|ListOfGoalLists]) :-
     catch(Module:verify_attributes(Var, Value, Goals),
@@ -33,6 +39,15 @@ verify_attrs([Module|Modules], Var, Value, [Goals|ListOfGoalLists]) :-
     verify_attrs(Modules, Var, Value, ListOfGoalLists).
 verify_attrs([], _, _, []).
 
+/*
+verify_attrs([Module|Modules], Var, Value, [Goals|ListOfGoalLists]) :-
+    catch(Module:verify_attributes(Var, Value, Goals),
+          error(evaluation_error((Module:verify_attributes)/3), verify_attributes/3),
+          Goals = []),
+    verify_attrs(Modules, Var, Value, ListOfGoalLists).
+verify_attrs([], _, _, []).
+*/
+
 call_goals([ListOfGoalLists | ListsCubed]) :-
     call_goals_0(ListOfGoalLists),
     call_goals(ListsCubed).

src/machine/attributed_variables.rs

@@ -6,9 +6,6 @@ use crate::indexmap::IndexSet;
 use std::cmp::Ordering;
 use std::vec::IntoIter;
 
-pub static VERIFY_ATTRS: &str = include_str!("attributed_variables.pl");
-pub static PROJECT_ATTRS: &str = include_str!("project_attributes.pl");
-
 pub(super) type Bindings = Vec<(usize, Addr)>;
 
 #[derive(Debug)]
@@ -126,7 +123,7 @@ impl MachineState {
         self.stack.index_and_frame_mut(e).prelude.interrupt_cp = self.attr_var_init.cp;
 
         for i in 1 .. self.num_of_args + 1 {
-            self.stack.index_and_frame_mut(e)[i] = self[RegType::Temp(i)].clone();
+            self.stack.index_and_frame_mut(e)[i] = self[RegType::Temp(i)];
         }
 
         self.stack.index_and_frame_mut(e)[self.num_of_args + 1] =

src/machine/code_repo.rs

@@ -1,127 +1,52 @@
 use crate::clause_types::*;
-use crate::codegen::*;
-use crate::debray_allocator::*;
-use crate::forms::*;
 use crate::instructions::*;
-use crate::machine::compile::*;
-use crate::machine::machine_errors::*;
 use crate::machine::machine_indices::*;
 
-use crate::indexmap::IndexSet;
-
-use std::collections::VecDeque;
-use std::mem;
-
 #[derive(Debug)]
 pub struct CodeRepo {
-    pub(super) cached_query: Code,
-    pub(super) goal_expanders: Code,
-    pub(super) term_expanders: Code,
     pub(super) code: Code,
-    pub(super) in_situ_code: Code,
-    pub(super) term_dir: TermDir,
 }
 
 impl CodeRepo {
     #[inline]
-    pub(super) fn new() -> Self {
+    pub(super)
+    fn new() -> Self {
         CodeRepo {
-            cached_query: vec![],
-            goal_expanders: Code::new(),
-            term_expanders: Code::new(),
             code: Code::new(),
-            in_situ_code: Code::new(),
-            term_dir: TermDir::new(),
         }
     }
 
     #[inline]
-    pub fn term_dir_entry_len(&self, key: PredicateKey) -> (usize, usize) {
-        self.term_dir
-            .get(&key)
-            .map(|entry| ((entry.0).0.len(), entry.1.len()))
-            .unwrap_or((0, 0))
-    }
-
-    #[inline]
-    pub fn truncate_terms(
-        &mut self,
-        key: PredicateKey,
-        len: usize,
-        queue_len: usize,
-    ) -> (Predicate, VecDeque<TopLevel>) {
-        self.term_dir
-            .get_mut(&key)
-            .map(|entry| {
-                let terms =
-                    if len < (entry.0).0.len() {
-                        (entry.0).0.drain(len ..).collect()
-                    } else {
-                        vec![]
-                    };
-
-                let queue =
-                    if queue_len < entry.1.len() {
-                        entry.1.drain(queue_len ..).collect()
-                    } else {
-                        VecDeque::new()
-                    };
-
-                (Predicate(terms), queue)
-            })
-            .unwrap_or((Predicate::new(), VecDeque::new()))
-    }
-
-    pub(crate)
-    fn add_in_situ_result(
-        &mut self,
-        result: &CompiledResult,
-        in_situ_code_dir: &mut InSituCodeDir,
-        in_situ_module_dir: &mut ModuleStubDir,
-        non_counted_bt_preds: &IndexSet<PredicateKey>,
-    ) -> Result<(), SessionError> {
-        let (ref decl, ref queue) = result;
-        let (name, arity) = decl
-            .0
-            .first()
-            .and_then(|cl| {
-                let arity = cl.arity();
-                cl.name().map(|name| (name, arity))
-            })
-            .ok_or(SessionError::NamelessEntry)?;
-
-        let non_counted_bt = non_counted_bt_preds.contains(&(name.clone(), arity));
-        let module_name = name.owning_module();
-
-        let p = self.in_situ_code.len();
-
-        match in_situ_module_dir.get_mut(&module_name) {
-            Some(ref mut module_stub) if name.has_table(&module_stub.atom_tbl) => {
-                module_stub.in_situ_code_dir.insert((name, arity), p);
+    pub(super)
+    fn lookup_local_instr<'a>(
+        &'a self,
+        p: LocalCodePtr,
+    ) -> RefOrOwned<'a, Line> {
+        match p {
+            LocalCodePtr::Halt => {
+                unreachable!()
             }
-            _ => {
-                in_situ_code_dir.insert((name, arity), p);
+            LocalCodePtr::DirEntry(p) => {
+                RefOrOwned::Borrowed(&self.code[p as usize])
+            }
+            LocalCodePtr::IndexingBuf(p, o, i) => {
+                match &self.code[p] {
+                    &Line::IndexingCode(ref indexing_lines) => {
+                        match &indexing_lines[o] {
+                            &IndexingLine::IndexedChoice(ref indexed_choice_instrs) => {
+                                RefOrOwned::Owned(Line::IndexedChoice(indexed_choice_instrs[i]))
+                            }
+                            _ => {
+                                unreachable!()
+                            }
+                        }
+                    }
+                    _ => {
+                        unreachable!()
+                    }
+                }
             }
         }
-
-        let mut cg = CodeGenerator::<DebrayAllocator>::new(non_counted_bt);
-        let mut decl_code = cg.compile_predicate(&decl.0)?;
-
-        compile_appendix(&mut decl_code, queue, non_counted_bt)?;
-
-        Ok(self.in_situ_code.extend(decl_code.into_iter()))
-    }
-
-    #[inline]
-    pub(super)
-    fn size_of_cached_query(&self) -> usize {
-        self.cached_query.len()
-    }
-
-    #[inline]
-    pub(super)
-    fn take_in_situ_code(&mut self) -> Code {
-        mem::replace(&mut self.in_situ_code, Code::new())
     }
 
     pub(super)
@@ -131,32 +56,12 @@ impl CodeRepo {
         p: &CodePtr,
     ) -> Option<RefOrOwned<'a, Line>> {
         match p {
-            &CodePtr::Local(LocalCodePtr::UserGoalExpansion(p)) => {
-                if p < self.goal_expanders.len() {
-                    Some(RefOrOwned::Borrowed(&self.goal_expanders[p]))
-                } else {
-                    None
-                }
+            &CodePtr::Local(local) => {
+                return Some(self.lookup_local_instr(local));
             }
-            &CodePtr::Local(LocalCodePtr::UserTermExpansion(p)) => {
-                if p < self.term_expanders.len() {
-                    Some(RefOrOwned::Borrowed(&self.term_expanders[p]))
-                } else {
-                    None
-                }
+            &CodePtr::REPL(..) => {
+                None
             }
-            &CodePtr::Local(LocalCodePtr::TopLevel(_, p)) => {
-                if p < self.cached_query.len() {
-                    Some(RefOrOwned::Borrowed(&self.cached_query[p]))
-                } else {
-                    None
-                }
-            }
-            &CodePtr::Local(LocalCodePtr::InSituDirEntry(p)) => {
-                Some(RefOrOwned::Borrowed(&self.in_situ_code[p]))
-            }
-            &CodePtr::Local(LocalCodePtr::DirEntry(p)) => Some(RefOrOwned::Borrowed(&self.code[p])),
-            &CodePtr::REPL(..) => None,
             &CodePtr::BuiltInClause(ref built_in, _) => {
                 let call_clause = call_clause!(
                     ClauseType::BuiltIn(built_in.clone()),
@@ -164,14 +69,27 @@ impl CodeRepo {
                     0,
                     last_call
                 );
+
                 Some(RefOrOwned::Owned(call_clause))
             }
             &CodePtr::CallN(arity, _, last_call) => {
-                let call_clause = call_clause!(ClauseType::CallN, arity, 0, last_call);
+                let call_clause = call_clause!(
+                    ClauseType::CallN,
+                    arity,
+                    0,
+                    last_call
+                );
+
                 Some(RefOrOwned::Owned(call_clause))
             }
-            &CodePtr::VerifyAttrInterrupt(p) => Some(RefOrOwned::Borrowed(&self.code[p])),
-            &CodePtr::DynamicTransaction(..) => None,
+            &CodePtr::VerifyAttrInterrupt(p) => {
+                Some(RefOrOwned::Borrowed(&self.code[p]))
+            }
+/*
+            &CodePtr::DynamicTransaction(..) => {
+                None
+            }
+*/
         }
     }
 }

src/machine/code_walker.rs

@@ -1,73 +1,39 @@
 use crate::instructions::*;
 
-use std::collections::VecDeque;
+use indexmap::IndexSet;
 
-fn scan_for_trust_me(
-    code: &Code,
-    jmp_offsets: &mut VecDeque<usize>,
-    before_idx: usize,
-    after_idx: &mut usize,
-) {
-    // record the location of the line after the TrustMe capping the
-    // choice instruction sequence to after_idx.
-    loop {
-        match &code[*after_idx] {
-            &Line::Choice(ChoiceInstruction::DefaultRetryMeElse(offset)) |
-            &Line::Choice(ChoiceInstruction::RetryMeElse(offset)) |
-            &Line::IndexedChoice(IndexedChoiceInstruction::Retry(offset)) => {
-                *after_idx += offset;
-            }
-            &Line::Choice(ChoiceInstruction::DefaultTrustMe) |
-            &Line::Choice(ChoiceInstruction::TrustMe) |
-            &Line::IndexedChoice(IndexedChoiceInstruction::Trust(..)) => {
-                break;
-            }
-            _ => {
-                *after_idx += 1;
+fn capture_offset(line: &Line, index: usize, stack: &mut Vec<usize>) -> bool {
+    match line {
+        &Line::Choice(ChoiceInstruction::TryMeElse(offset)) if offset > 0 => {
+            stack.push(index + offset);
+        }
+        &Line::Choice(ChoiceInstruction::DefaultRetryMeElse(offset)) |
+        &Line::Choice(ChoiceInstruction::RetryMeElse(offset)) if offset > 0 => {
+            stack.push(index + offset);
+        }
+        &Line::Control(ControlInstruction::JmpBy(_, offset, _, false)) => {
+            stack.push(index + offset);
+        }
+        &Line::Control(ControlInstruction::JmpBy(_, offset, _, true)) => {
+            stack.push(index + offset);
+            return true;
+        }
+        &Line::Control(ControlInstruction::Proceed) |
+        &Line::Control(ControlInstruction::CallClause(_, _, _, true, _)) => {
+            return true;
+        }
+        &Line::Control(ControlInstruction::RevJmpBy(offset)) => {
+            if offset > 0 {
+                stack.push(index - offset);
+            } else {
+                return true;
             }
         }
-    }
-
-    // search the code in the range for JmpBy instructions and record their
-    // offsets for future scanning.
-    for (idx, instr) in code[before_idx .. *after_idx].iter().enumerate() {
-        match instr {
-            &Line::Control(ControlInstruction::JmpBy(_, offset, ..)) => {
-                jmp_offsets.push_back(before_idx + idx + offset)
-            }
-            _ => {
-            }
+        _ => {
         }
-    }
+    };
 
-    *after_idx += 1;
-}
-
-fn capture_next_range(code: &Code, queue: &mut VecDeque<usize>, last_idx: &mut usize) {
-    loop {
-        match &code[*last_idx] {
-            &Line::Choice(ChoiceInstruction::TryMeElse(offset)) |
-            &Line::IndexedChoice(IndexedChoiceInstruction::Try(offset)) => {
-                let before_idx = *last_idx;
-                *last_idx += offset;
-
-                scan_for_trust_me(code, queue, before_idx, last_idx);
-            }
-            &Line::Control(ControlInstruction::JmpBy(_, offset, _, false)) => {
-                queue.push_back(*last_idx + offset);
-                *last_idx += 1;
-            }
-            &Line::Control(ControlInstruction::JmpBy(_, offset, _, true)) => {
-                queue.push_back(*last_idx + offset);
-                break;
-            }
-            &Line::Control(ControlInstruction::Proceed) |
-            &Line::Control(ControlInstruction::CallClause(_, _, _, true, _)) =>
-                break,
-            _ =>
-                *last_idx += 1,
-        };
-    }
+    false
 }
 
 /* This function walks the code of a single predicate, supposed to
@@ -76,15 +42,22 @@ fn capture_next_range(code: &Code, queue: &mut VecDeque<usize>, last_idx: &mut u
  */
 pub fn walk_code(code: &Code, p: usize, mut walker: impl FnMut(&Line))
 {
-    let mut queue = VecDeque::from(vec![p]);
+    let mut stack = vec![p];
+    let mut visited_indices = IndexSet::new();
 
-    while let Some(first_idx) = queue.pop_front() {
-        let mut last_idx = first_idx;
+    while let Some(first_index) = stack.pop() {
+        if visited_indices.contains(&first_index) {
+            continue;
+        } else {
+            visited_indices.insert(first_index);
+        }
 
-        capture_next_range(code, &mut queue, &mut last_idx);
-
-        for instr in &code[first_idx .. last_idx + 1] {
+        for (index, instr) in code[first_index ..].iter().enumerate() {
             walker(instr);
+
+            if capture_offset(instr, first_index + index, &mut stack) {
+                break;
+            }
         }
     }
 }
@@ -92,6 +65,7 @@ pub fn walk_code(code: &Code, p: usize, mut walker: impl FnMut(&Line))
 /* A function for code walking that might result in modification to
  * the code. Otherwise identical to walk_code.
  */
+/*
 pub fn walk_code_mut(code: &mut Code, p: usize, mut walker: impl FnMut(&mut Line))
 {
     let mut queue = VecDeque::from(vec![p]);
@@ -106,3 +80,4 @@ pub fn walk_code_mut(code: &mut Code, p: usize, mut walker: impl FnMut(&mut Line
         }
     }
 }
+*/

src/machine/dynamic_database.rs

@@ -1,396 +0,0 @@
-use crate::prolog_parser::ast::*;
-
-use crate::heap_print::*;
-use crate::machine::*;
-use crate::machine::compile::*;
-use crate::machine::machine_errors::*;
-use crate::machine::streams::*;
-
-use std::convert::TryFrom;
-
-impl Machine {
-    pub(super) fn atom_tbl_of(&self, name: &ClauseName) -> TabledData<Atom> {
-        match name {
-            &ClauseName::User(ref rc) => rc.table.clone(),
-            _ => self.indices.atom_tbl(),
-        }
-    }
-
-    fn compile_into_machine(
-        &mut self,
-        src: Stream,
-        name: ClauseName,
-        arity: usize,
-    ) -> EvalSession {
-        match name.owning_module().as_str() {
-            "user" => match self.indices.code_dir.get(&(name.clone(), arity)).cloned() {
-                Some(idx) => {
-                    let module = idx.0.borrow().1.clone();
-
-                    match module.as_str() {
-                        "user" => compile_user_module(self, src, true, ListingSource::User),
-                        _ => compile_into_module(self, module, src, name)
-                    }
-                }
-                None => compile_user_module(self, src, true, ListingSource::User),
-            },
-            _ => compile_into_module(self, name.owning_module(), src, name),
-        }
-    }
-
-    fn get_predicate_key(&self, name: RegType, arity: RegType) -> PredicateKey {
-        let name = self.machine_st[name].clone();
-        let arity = self.machine_st[arity].clone();
-
-        let name = match self.machine_st.store(self.machine_st.deref(name)) {
-            Addr::Con(h) =>
-                if let HeapCellValue::Atom(ref name, _) = &self.machine_st.heap[h] {
-                    name.clone()
-                } else {
-                    unreachable!()
-                },
-            _ => unreachable!(),
-        };
-
-        let arity = match self.machine_st.store(self.machine_st.deref(arity)) {
-            Addr::Con(h) => {
-                match &self.machine_st.heap[h] {
-                    HeapCellValue::Integer(ref arity) => {
-                        arity.to_usize().unwrap()
-                    }
-                    HeapCellValue::Addr(Addr::Fixnum(arity)) => {
-                        usize::try_from(*arity).unwrap()
-                    }
-                    _ => {
-                        unreachable!()
-                    }
-                }
-            }
-            Addr::Fixnum(arity) => {
-                usize::try_from(arity).unwrap()
-            }
-            Addr::Usize(n) => {
-                n
-            }
-            _ => {
-                unreachable!()
-            }
-        };
-
-        (name, arity)
-    }
-
-    fn print_new_dynamic_clause(
-        &self,
-        addrs: VecDeque<Addr>,
-        name: ClauseName,
-        arity: usize,
-    ) -> String {
-        let mut output = PrinterOutputter::new();
-        output.append(format!(":- dynamic({}/{}). ", name.as_str(), arity).as_str());
-
-        for addr in addrs {
-            let mut printer = HCPrinter::new(&self.machine_st, &self.indices.op_dir, output);
-            printer.quoted = true;
-
-            output = printer.print(addr);
-            output.append(". ");
-        }
-
-        output.result()
-    }
-
-    fn make_undefined(&mut self, name: ClauseName, arity: usize) {
-        let module_name = name.owning_module();
-
-        match self.indices.modules.get(&module_name) {
-            Some(ref module) => {
-                if let Some(idx) = module.code_dir.get(&(name.clone(), arity)) {
-                    set_code_index!(idx, IndexPtr::DynamicUndefined, module_name);
-                }
-            }
-            None => {
-            }
-        }
-
-        if let Some(idx) = self.indices.code_dir.get(&(name, arity)) {
-            set_code_index!(idx, IndexPtr::DynamicUndefined, clause_name!("user"));
-        }
-    }
-
-    fn make_undefined_in_module(&mut self, module_name: ClauseName, name: ClauseName, arity: usize) {
-        if let Some(idx) = self.indices.code_dir.get(&(name, arity)) {
-            if idx.module_name() == module_name {
-                set_code_index!(idx, IndexPtr::DynamicUndefined, clause_name!("user"));
-            }
-        }
-    }
-
-    fn abolish_dynamic_clause(&mut self, name: RegType, arity: RegType) {
-        let (name, arity) = self.get_predicate_key(name, arity);
-
-        self.make_undefined(name.clone(), arity);
-
-        self.indices.remove_code_index((name.clone(), arity));
-        self.indices.remove_clause_subsection(name.owning_module(), name, arity);
-    }
-
-    fn abolish_dynamic_clause_in_module(&mut self, name: RegType, arity: RegType, module: RegType) {
-        let (name, arity) = self.get_predicate_key(name, arity);
-        let module_addr = self.machine_st[module].clone();
-
-        let module_name = match self.machine_st.store(self.machine_st.deref(module_addr)) {
-            Addr::Con(h) =>
-                if let HeapCellValue::Atom(ref module, _) = &self.machine_st.heap[h] {
-                    match self.indices.modules.get_mut(module) {
-                        Some(ref mut module) => {
-                            module.code_dir.remove(&(name.clone(), arity));
-                            module.module_decl.name.clone()
-                        }
-                        _ => {
-                            self.machine_st.fail = true;
-                            return;
-                        }
-                    }
-                } else {
-                    unreachable!()
-                },
-            _ => unreachable!(),
-        };
-
-        self.make_undefined_in_module(module_name.clone(), name.clone(), arity);
-
-        self.indices.remove_code_index((name.clone(), arity));
-        self.indices.remove_clause_subsection(module_name, name, arity);
-    }
-
-    fn handle_eval_result_from_dynamic_compile(
-        &mut self,
-        pred_str: String,
-        name: ClauseName,
-        arity: usize,
-        src: ClauseName,
-    ) {
-        let machine_st = mem::replace(&mut self.machine_st, MachineState::new());
-
-        let result = self.compile_into_machine(
-            Stream::from(pred_str),
-            name,
-            arity,
-        );
-
-        self.machine_st = machine_st;
-
-        if let EvalSession::Error(err) = result {
-            let h = self.machine_st.heap.h();
-            let stub = MachineError::functor_stub(src, 1);
-            let err = MachineError::session_error(h, err);
-            let err = self.machine_st.error_form(err, stub);
-
-            self.machine_st.throw_exception(err);
-        }
-    }
-
-    fn recompile_dynamic_predicate_impl(
-        &mut self,
-        place: DynamicAssertPlace,
-        name: ClauseName,
-        arity: usize,
-    ) {
-        let stub = MachineError::functor_stub(place.predicate_name(), 1);
-        let pred_str = match self.machine_st.try_from_list(temp_v!(2), stub) {
-            Ok(addrs) => {
-                let mut addrs = VecDeque::from(addrs);
-                let added_clause = self.machine_st[temp_v!(1)].clone();
-
-                place.push_to_queue(&mut addrs, added_clause);
-                self.print_new_dynamic_clause(addrs, name.clone(), arity)
-            }
-            Err(err) => {
-                return self.machine_st.throw_exception(err);
-            }
-        };
-
-        self.handle_eval_result_from_dynamic_compile(
-            pred_str,
-            name,
-            arity,
-            place.predicate_name(),
-        );
-    }
-
-    fn set_module_atom_tbl(&mut self, module_addr: Addr, name: &mut ClauseName) -> bool {
-        let atom_tbl = match self.machine_st.store(self.machine_st.deref(module_addr)) {
-            Addr::Con(h) =>
-                if let HeapCellValue::Atom(ref module, _) = &self.machine_st.heap[h] {
-                    match self.indices.modules.get(module) {
-                        Some(ref module) => module.atom_tbl.clone(),
-                        None => {
-                            self.machine_st.fail = true;
-                            return false;
-                        }
-                    }
-                } else {
-                    self.machine_st.fail = true;
-                    return false;
-                },
-            _ => unreachable!(),
-        };
-
-        if let &mut ClauseName::User(ref mut rc) = name {
-            rc.table = atom_tbl;
-        }
-
-        true
-    }
-
-    fn recompile_dynamic_predicate_in_module(&mut self, place: DynamicAssertPlace) {
-        let (mut name, arity) = self.get_predicate_key(temp_v!(3), temp_v!(4));
-        let module_addr = self.machine_st[temp_v!(5)].clone();
-
-        if self.set_module_atom_tbl(module_addr, &mut name) {
-            self.recompile_dynamic_predicate_impl(place, name, arity);
-        }
-    }
-
-    fn recompile_dynamic_predicate(&mut self, place: DynamicAssertPlace) {
-        let (name, arity) = self.get_predicate_key(temp_v!(3), temp_v!(4));
-        self.recompile_dynamic_predicate_impl(place, name, arity);
-    }
-
-    fn retract_from_dynamic_predicate_in_module(&mut self) {
-        let index = self.machine_st[temp_v!(3)].clone();
-        let index = match self.machine_st.store(self.machine_st.deref(index)) {
-            Addr::Con(h) =>
-                match &self.machine_st.heap[h] {
-                    HeapCellValue::Integer(ref arity) => {
-                        arity.to_usize().unwrap()
-                    }
-                    HeapCellValue::Addr(Addr::Fixnum(arity)) => {
-                        usize::try_from(*arity).unwrap()
-                    }
-                    _ => {
-                        unreachable!()
-                    }
-                }
-            Addr::Fixnum(arity) => {
-                usize::try_from(arity).unwrap()
-            }
-            _ => {
-                unreachable!()
-            }
-        };
-
-        let (mut name, arity) = self.get_predicate_key(temp_v!(1), temp_v!(2));
-        let module_addr = self.machine_st[temp_v!(5)].clone();
-
-        if self.set_module_atom_tbl(module_addr, &mut name) {
-            let stub = MachineError::functor_stub(clause_name!("retract"), 1);
-            let pred_str = match self.machine_st.try_from_list(temp_v!(4), stub) {
-                Ok(addrs) => {
-                    let mut addrs = VecDeque::from(addrs);
-                    addrs.remove(index);
-
-                    if addrs.is_empty() {
-                        self.make_undefined(name.clone(), arity);
-                    }
-
-                    self.print_new_dynamic_clause(addrs, name.clone(), arity)
-                }
-                Err(err) => {
-                    return self.machine_st.throw_exception(err);
-                }
-            };
-
-            self.handle_eval_result_from_dynamic_compile(
-                pred_str,
-                name,
-                arity,
-                clause_name!("retract"),
-            );
-        }
-    }
-
-    fn retract_from_dynamic_predicate(&mut self) {
-        let index = self.machine_st[temp_v!(3)].clone();
-        let index = match self.machine_st.store(self.machine_st.deref(index)) {
-            Addr::Con(h) => {
-                match &self.machine_st.heap[h] {
-                    HeapCellValue::Integer(ref arity) => {
-                        arity.to_usize().unwrap()
-                    }
-                    HeapCellValue::Addr(Addr::Fixnum(arity)) => {
-                        usize::try_from(*arity).unwrap()
-                    }
-                    _ => {
-                        unreachable!()
-                    }
-                }
-            }
-            Addr::Usize(n) => {
-                n
-            }
-            Addr::Fixnum(n) => {
-                usize::try_from(n).unwrap()
-            }
-            _ => {
-                unreachable!()
-            }
-        };
-
-        let (name, arity) = self.get_predicate_key(temp_v!(1), temp_v!(2));
-
-        let stub = MachineError::functor_stub(clause_name!("retract"), 1);
-        let pred_str = match self.machine_st.try_from_list(temp_v!(4), stub) {
-            Ok(addrs) => {
-                let mut addrs = VecDeque::from(addrs);
-                addrs.remove(index);
-
-                if addrs.is_empty() {
-                    self.make_undefined(name.clone(), arity);
-                }
-
-                self.print_new_dynamic_clause(addrs, name.clone(), arity)
-            }
-            Err(err) => {
-                return self.machine_st.throw_exception(err);
-            }
-        };
-
-        self.handle_eval_result_from_dynamic_compile(
-            pred_str,
-            name,
-            arity,
-            clause_name!("retract"),
-        );
-    }
-
-    pub(super) fn dynamic_transaction(
-        &mut self,
-        trans_type: DynamicTransactionType,
-        p: LocalCodePtr,
-    ) {
-        match trans_type {
-            DynamicTransactionType::Abolish => {
-                self.abolish_dynamic_clause(temp_v!(1), temp_v!(2))
-            }
-            DynamicTransactionType::Assert(place) => {
-                self.recompile_dynamic_predicate(place)
-            }
-            DynamicTransactionType::ModuleAbolish => {
-                self.abolish_dynamic_clause_in_module(temp_v!(1), temp_v!(2), temp_v!(3))
-            }
-            DynamicTransactionType::ModuleAssert(place) => {
-                self.recompile_dynamic_predicate_in_module(place)
-            }
-            DynamicTransactionType::ModuleRetract => {
-                self.retract_from_dynamic_predicate_in_module()
-            }
-            DynamicTransactionType::Retract => {
-                self.retract_from_dynamic_predicate()
-            }
-        }
-
-        self.machine_st.p = CodePtr::Local(p);
-    }
-}

src/machine/heap.rs

@@ -168,6 +168,9 @@ impl<T: RawBlockTraits> HeapTemplate<T> {
             &HeapCellValue::Integer(ref n) => {
                 HeapCellValue::Integer(n.clone())
             }
+            &HeapCellValue::LoadStatePayload(_) => {
+                HeapCellValue::Addr(Addr::LoadStatePayload(h))
+            }
             &HeapCellValue::NamedStr(arity, ref name, ref op) => {
                 HeapCellValue::NamedStr(arity, name.clone(), op.clone())
             }
@@ -295,6 +298,9 @@ impl<T: RawBlockTraits> HeapTemplate<T> {
             val @ HeapCellValue::Rational(_) => {
                 Addr::Con(self.push(val))
             }
+            val @ HeapCellValue::LoadStatePayload(_) => {
+                Addr::LoadStatePayload(self.push(val))
+            }
             val @ HeapCellValue::NamedStr(..) => {
                 Addr::Str(self.push(val))
             }
@@ -371,15 +377,6 @@ impl<T: RawBlockTraits> HeapTemplate<T> {
         }
     }
 
-    #[inline]
-    pub(crate)
-    fn take(&mut self) -> Self {
-        HeapTemplate {
-            buf: self.buf.take(),
-            _marker: PhantomData,
-        }
-    }
-
     #[inline]
     pub(crate)
     fn truncate(&mut self, h: usize) {
@@ -479,9 +476,7 @@ impl<T: RawBlockTraits> HeapTemplate<T> {
                             ("dir_entry", 1) => {
                                 extract_integer(s+1).map(LocalCodePtr::DirEntry)
                             }
-                            ("in_situ_dir_entry", 1) => {
-                                extract_integer(s+1).map(LocalCodePtr::InSituDirEntry)
-                            }
+                            /*
                             ("top_level", 2) => {
                                 if let Some(chunk_num) = extract_integer(s+1) {
                                     if let Some(p) = extract_integer(s+2) {
@@ -491,13 +486,10 @@ impl<T: RawBlockTraits> HeapTemplate<T> {
 
                                 None
                             }
-                            ("user_goal_expansion", 1) => {
-                                extract_integer(s+1).map(LocalCodePtr::UserGoalExpansion)
+                            */
+                            _ => {
+                                None
                             }
-                            ("user_term_expansion", 1) => {
-                                extract_integer(s+1).map(LocalCodePtr::UserTermExpansion)
-                            }
-                            _ => None
                         }
                     }
                     _ => unreachable!()
@@ -508,8 +500,7 @@ impl<T: RawBlockTraits> HeapTemplate<T> {
     }
 
     #[inline]
-    pub
-    fn index_addr<'a>(&'a self, addr: &Addr) -> RefOrOwned<'a, HeapCellValue> {
+    pub fn index_addr<'a>(&'a self, addr: &Addr) -> RefOrOwned<'a, HeapCellValue> {
         match addr {
             &Addr::Con(h) | &Addr::Str(h) | &Addr::Stream(h) | &Addr::TcpListener(h) => {
                 RefOrOwned::Borrowed(&self[h])

src/machine/load_state.rs

@@ -0,0 +1,918 @@
+use crate::machine::*;
+use crate::machine::machine_indices::*;
+use crate::machine::term_stream::*;
+
+use indexmap::IndexSet;
+
+use crate::ref_thread_local::RefThreadLocal;
+
+type ModuleOpExports = Vec<(OpDecl, Option<(usize, Specifier)>)>;
+
+/*
+ * We will want to borrow these fields from Loader separately, without
+ * restricting access to other fields by borrowing them mutably.
+ */
+pub(super) struct LoadState<'a> {
+    pub(super) compilation_target: CompilationTarget,
+    pub(super) module_op_exports: ModuleOpExports,
+    pub(super) retraction_info: RetractionInfo,
+    pub(super) wam: &'a mut Machine,
+}
+
+pub(super)
+fn set_code_index(
+    retraction_info: &mut RetractionInfo,
+    compilation_target: &CompilationTarget,
+    key: PredicateKey,
+    code_index: &CodeIndex,
+    code_ptr: IndexPtr,
+) {
+    let record =
+        match compilation_target {
+            CompilationTarget::User => {
+                if IndexPtr::Undefined == code_index.get() {
+                    code_index.set(code_ptr);
+                    RetractionRecord::AddedUserPredicate(key)
+                } else {
+                    // TODO: emit warning about overwriting previous record
+                    let replaced = code_index.replace(code_ptr);
+                    RetractionRecord::ReplacedUserPredicate(key, replaced)
+                }
+            }
+            CompilationTarget::Module(ref module_name) => {
+                if IndexPtr::Undefined == code_index.get() {
+                    code_index.set(code_ptr);
+                    RetractionRecord::AddedModulePredicate(module_name.clone(), key)
+                } else {
+                    // TODO: emit warning about overwriting previous record
+                    let replaced = code_index.replace(code_ptr);
+                    RetractionRecord::ReplacedModulePredicate(module_name.clone(), key, replaced)
+                }
+            }
+        };
+
+    retraction_info.push_record(record);
+}
+
+fn add_op_decl_as_module_export(
+    module_op_dir: &mut OpDir,
+    compilation_target: &CompilationTarget,
+    retraction_info: &mut RetractionInfo,
+    wam_op_dir: &mut OpDir,
+    module_op_exports: &mut ModuleOpExports,
+    op_decl: &OpDecl,
+) {
+    /*
+    insert the operator at top-level so it can
+    inform the parser. it will be retracted
+    from the user-level op_dir when the load
+    succeeds.
+    */
+
+    match op_decl.insert_into_op_dir(wam_op_dir) {
+        Some((prec, spec)) => {
+            retraction_info.push_record(
+                RetractionRecord::ReplacedUserOp(op_decl.clone(), prec, spec)
+            );
+
+            module_op_exports.push((op_decl.clone(), Some((prec, spec))));
+        }
+        None => {
+            retraction_info.push_record(
+                RetractionRecord::AddedUserOp(op_decl.clone())
+            );
+
+            module_op_exports.push((op_decl.clone(), None));
+        }
+    }
+
+    add_op_decl(retraction_info, compilation_target, module_op_dir, op_decl);
+}
+
+pub(super)
+fn add_op_decl(
+    retraction_info: &mut RetractionInfo,
+    compilation_target: &CompilationTarget,
+    op_dir: &mut OpDir,
+    op_decl: &OpDecl,
+) {
+    match op_decl.insert_into_op_dir(op_dir) {
+        Some((prec, spec)) => {
+            match &compilation_target {
+                CompilationTarget::User => {
+                    retraction_info.push_record(
+                        RetractionRecord::ReplacedUserOp(op_decl.clone(), prec, spec),
+                    );
+                }
+                CompilationTarget::Module(ref module_name) => {
+                    retraction_info.push_record(
+                        RetractionRecord::ReplacedModuleOp(
+                            module_name.clone(), op_decl.clone(), prec, spec,
+                        ),
+                    );
+                }
+            }
+        }
+        None => {
+            match &compilation_target {
+                CompilationTarget::User => {
+                    retraction_info.push_record(
+                        RetractionRecord::AddedUserOp(op_decl.clone()),
+                    );
+                }
+                CompilationTarget::Module(ref module_name) => {
+                    retraction_info.push_record(
+                        RetractionRecord::AddedModuleOp(module_name.clone(), op_decl.clone()),
+                    );
+                }
+            }
+        }
+    }
+}
+
+pub(super)
+fn import_module_exports(
+    retraction_info: &mut RetractionInfo,
+    compilation_target: &CompilationTarget,
+    imported_module: &Module,
+    code_dir: &mut CodeDir,
+    op_dir: &mut OpDir,
+    meta_predicates: &mut MetaPredicateDir,
+) {
+    for export in imported_module.module_decl.exports.iter() {
+        match export {
+            ModuleExport::PredicateKey((ref name, arity)) => {
+                let key = (name.clone(), *arity);
+
+                if let Some(meta_specs) = imported_module.meta_predicates.get(&key) {
+                    meta_predicates.insert(key.clone(), meta_specs.clone());
+                }
+
+                if let Some(src_code_index) = imported_module.code_dir.get(&key) {
+                    let target_code_index = code_dir
+                        .entry(key.clone())
+                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .clone();
+
+                    set_code_index(
+                        retraction_info,
+                        compilation_target,
+                        key,
+                        &target_code_index,
+                        src_code_index.get(),
+                    );
+                } else {
+                    unreachable!()
+                }
+            }
+            ModuleExport::OpDecl(ref op_decl) => {
+                add_op_decl(
+                    retraction_info,
+                    compilation_target,
+                    op_dir,
+                    op_decl,
+                );
+            }
+        }
+    }
+}
+
+fn import_module_exports_into_module(
+    retraction_info: &mut RetractionInfo,
+    compilation_target: &CompilationTarget,
+    imported_module: &Module,
+    code_dir: &mut CodeDir,
+    op_dir: &mut OpDir,
+    meta_predicates: &mut MetaPredicateDir,
+    wam_op_dir: &mut OpDir,
+    module_op_exports: &mut ModuleOpExports
+) {
+    for export in imported_module.module_decl.exports.iter() {
+        match export {
+            ModuleExport::PredicateKey((ref name, arity)) => {
+                let key = (name.clone(), *arity);
+
+                if let Some(meta_specs) = imported_module.meta_predicates.get(&key) {
+                    meta_predicates.insert(key.clone(), meta_specs.clone());
+                }
+
+                if let Some(src_code_index) = imported_module.code_dir.get(&key) {
+                    let target_code_index = code_dir
+                        .entry(key.clone())
+                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .clone();
+
+                    set_code_index(
+                        retraction_info,
+                        compilation_target,
+                        key,
+                        &target_code_index,
+                        src_code_index.get(),
+                    );
+                } else {
+                    unreachable!()
+                }
+            }
+            ModuleExport::OpDecl(ref op_decl) => {
+                add_op_decl_as_module_export(
+                    op_dir,
+                    compilation_target,
+                    retraction_info,
+                    wam_op_dir,
+                    module_op_exports,
+                    op_decl,
+                );
+            }
+        }
+    }
+}
+
+
+fn import_qualified_module_exports(
+    retraction_info: &mut RetractionInfo,
+    compilation_target: &CompilationTarget,
+    imported_module: &Module,
+    exports: &IndexSet<ModuleExport>,
+    code_dir: &mut CodeDir,
+    op_dir: &mut OpDir,
+) {
+    for export in imported_module.module_decl.exports.iter() {
+        if !exports.contains(export) {
+            continue;
+        }
+
+        match export {
+            ModuleExport::PredicateKey((ref name, arity)) => {
+                let key = (name.clone(), *arity);
+
+                if let Some(src_code_index) = imported_module.code_dir.get(&key) {
+                    let target_code_index = code_dir
+                        .entry(key.clone())
+                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .clone();
+
+                    set_code_index(
+                        retraction_info,
+                        compilation_target,
+                        key,
+                        &target_code_index,
+                        src_code_index.get(),
+                    );
+                } else {
+                    unreachable!()
+                }
+            }
+            ModuleExport::OpDecl(ref op_decl) => {
+                add_op_decl(
+                    retraction_info,
+                    compilation_target,
+                    op_dir,
+                    op_decl,
+                );
+            }
+        }
+    }
+}
+
+fn import_qualified_module_exports_into_module(
+    retraction_info: &mut RetractionInfo,
+    compilation_target: &CompilationTarget,
+    imported_module: &Module,
+    exports: &IndexSet<ModuleExport>,
+    code_dir: &mut CodeDir,
+    op_dir: &mut OpDir,
+    wam_op_dir: &mut OpDir,
+    module_op_exports: &mut ModuleOpExports,
+) {
+    for export in imported_module.module_decl.exports.iter() {
+        if !exports.contains(export) {
+            continue;
+        }
+
+        match export {
+            ModuleExport::PredicateKey((ref name, arity)) => {
+                let key = (name.clone(), *arity);
+
+                if let Some(src_code_index) = imported_module.code_dir.get(&key) {
+                    let target_code_index = code_dir
+                        .entry(key.clone())
+                        .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                        .clone();
+
+                    set_code_index(
+                        retraction_info,
+                        compilation_target,
+                        key,
+                        &target_code_index,
+                        src_code_index.get(),
+                    );
+                } else {
+                    unreachable!()
+                }
+            }
+            ModuleExport::OpDecl(ref op_decl) => {
+                add_op_decl_as_module_export(
+                    op_dir,
+                    compilation_target,
+                    retraction_info,
+                    wam_op_dir,
+                    module_op_exports,
+                    op_decl,
+                );
+            }
+        }
+    }
+}
+
+impl<'a> LoadState<'a> {
+    #[inline]
+    pub(super)
+    fn increment_clause_assert_margin(&mut self, incr: usize) {
+        match &self.compilation_target {
+            CompilationTarget::User => {
+            }
+            CompilationTarget::Module(ref module_name) => {
+                self.retraction_info.push_record(
+                    RetractionRecord::IncreasedClauseAssertMargin(
+                        module_name.clone(),
+                        incr,
+                    ),
+                );
+
+                self.wam.indices.modules.get_mut(module_name)
+                    .map(|module| module.clause_assert_margin += incr);
+            }
+        }
+    }
+
+    #[inline]
+    pub(super)
+    fn remove_module_op_exports(&mut self) {
+        for (mut op_decl, record) in self.module_op_exports.drain(0 ..) {
+            op_decl.remove(&mut self.wam.indices.op_dir);
+
+            if let Some((prec, spec)) = record {
+                op_decl.prec = prec;
+                op_decl.spec = spec;
+                op_decl.insert_into_op_dir(&mut self.wam.indices.op_dir);
+            }
+        }
+    }
+
+    fn get_or_insert_local_code_index(
+        &mut self,
+        module_name: ClauseName,
+        key: PredicateKey,
+    ) -> CodeIndex {
+        match self.wam.indices.modules.get_mut(&module_name) {
+            Some(ref mut module) => {
+                module.code_dir
+                    .entry(key)
+                    .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                    .clone()
+            }
+            None => {
+                let mut module = Module::new(
+                    ModuleDecl { name: module_name.clone(), exports: vec![] },
+                    ListingSource::DynamicallyGenerated,
+                );
+
+                let code_index = module.code_dir
+                    .entry(key)
+                    .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                    .clone();
+
+                self.retraction_info.push_record(
+                    RetractionRecord::AddedModule(module_name.clone()),
+                );
+
+                self.wam.indices.modules.insert(module_name, module);
+                code_index
+            }
+        }
+    }
+
+    pub(super)
+    fn get_or_insert_code_index(&mut self, key: PredicateKey) -> CodeIndex {
+        match self.compilation_target.clone() {
+            CompilationTarget::User => {
+                self.wam.indices.code_dir
+                    .entry(key)
+                    .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                    .clone()
+            }
+            CompilationTarget::Module(module_name) => {
+                self.get_or_insert_local_code_index(module_name, key)
+            }
+        }
+    }
+
+    pub(super)
+    fn get_or_insert_qualified_code_index(
+        &mut self,
+        module_name: ClauseName,
+        key: PredicateKey,
+    ) -> CodeIndex {
+        if module_name.as_str() == "user" {
+            return self.wam.indices.code_dir
+                .entry(key)
+                .or_insert_with(|| CodeIndex::new(IndexPtr::Undefined))
+                .clone();
+        } else {
+            self.get_or_insert_local_code_index(module_name, key)
+        }
+    }
+
+    #[inline]
+    pub(super)
+    fn add_extensible_predicate(&mut self, key: PredicateKey, skeleton: PredicateSkeleton) {
+        match &self.compilation_target {
+            CompilationTarget::User => {
+                self.wam.indices.extensible_predicates.insert(key.clone(), skeleton);
+
+                self.retraction_info.push_record(
+                    RetractionRecord::AddedUserExtensiblePredicate(key),
+                );
+            }
+            CompilationTarget::Module(ref module_name) => {
+                if let Some(module) = self.wam.indices.modules.get_mut(module_name) {
+                    module.extensible_predicates.insert(key.clone(), skeleton);
+
+                    self.retraction_info.push_record(
+                        RetractionRecord::AddedModuleExtensiblePredicate(module_name.clone(), key),
+                    );
+                } else {
+                    unreachable!()
+                }
+            }
+        }
+    }
+
+    pub(super)
+    fn add_op_decl(&mut self, op_decl: &OpDecl) {
+        match &self.compilation_target {
+            CompilationTarget::User => {
+                add_op_decl(
+                    &mut self.retraction_info,
+                    &self.compilation_target,
+                    &mut self.wam.indices.op_dir,
+                    op_decl,
+                );
+            }
+            CompilationTarget::Module(ref module_name) => {
+                match self.wam.indices.modules.get_mut(module_name) {
+                    Some(ref mut module) => {
+                        add_op_decl_as_module_export(
+                            &mut module.op_dir,
+                            &self.compilation_target,
+                            &mut self.retraction_info,
+                            &mut self.wam.indices.op_dir,
+                            &mut self.module_op_exports,
+                            op_decl,
+                        );
+                    }
+                    _ => {
+                        unreachable!()
+                    }
+                }
+            }
+        }
+    }
+
+    pub(super)
+    fn get_clause_type(
+        &mut self,
+        name: ClauseName,
+        arity: usize,
+        fixity: Option<SharedOpDesc>,
+    ) -> ClauseType {
+        match ClauseType::from(name, arity, fixity) {
+            ClauseType::Named(name, arity, _) => {
+                let idx = self.get_or_insert_code_index((name.clone(), arity));
+                ClauseType::Named(name, arity, idx)
+            }
+            ClauseType::Op(name, fixity, _) => {
+                let idx = self.get_or_insert_code_index((name.clone(), arity));
+                ClauseType::Op(name, fixity, idx)
+            }
+            ct => {
+                ct
+            }
+        }
+    }
+
+    pub(super)
+    fn get_qualified_clause_type(
+        &mut self,
+        module_name: ClauseName,
+        name: ClauseName,
+        arity: usize,
+        fixity: Option<SharedOpDesc>,
+    ) -> ClauseType {
+        match ClauseType::from(name, arity, fixity) {
+            ClauseType::Named(name, arity, _) => {
+                let key = (name.clone(), arity);
+                let idx = self.get_or_insert_qualified_code_index(module_name, key);
+
+                ClauseType::Named(name, arity, idx)
+            }
+            ClauseType::Op(name, fixity, _) => {
+                let key = (name.clone(), arity);
+                let idx = self.get_or_insert_qualified_code_index(module_name, key);
+
+                ClauseType::Op(name, fixity, idx)
+            }
+            ct => {
+                ct
+            }
+        }
+    }
+
+    #[inline]
+    pub(super)
+    fn module_name(&self) -> ClauseName {
+        match self.compilation_target {
+            CompilationTarget::User => {
+                clause_name!("user")
+            }
+            CompilationTarget::Module(ref module_name) => {
+                module_name.clone()
+            }
+        }
+    }
+
+    pub(super)
+    fn add_meta_predicate_record(
+        &mut self,
+        module_name: ClauseName,
+        name: ClauseName,
+        meta_specs: Vec<MetaSpec>,
+    ) {
+        let arity = meta_specs.len();
+        let key = (name, arity);
+
+        match module_name.as_str() {
+            "user" => {
+                match self.wam.indices.meta_predicates.insert(key.clone(), meta_specs) {
+                    Some(old_meta_specs) => {
+                        self.retraction_info.push_record(
+                            RetractionRecord::ReplacedMetaPredicate(
+                                module_name.clone(), key.0, old_meta_specs,
+                            ),
+                        );
+                    }
+                    None => {
+                        self.retraction_info.push_record(
+                            RetractionRecord::AddedMetaPredicate(
+                                module_name.clone(), key,
+                            )
+                        );
+                    }
+                }
+            }
+            _ => {
+                match self.wam.indices.modules.get_mut(&module_name) {
+                    Some(ref mut module) => {
+                        match module.meta_predicates.insert(key.clone(), meta_specs) {
+                            Some(old_meta_specs) => {
+                                self.retraction_info.push_record(
+                                    RetractionRecord::ReplacedMetaPredicate(
+                                        module_name.clone(), key.0, old_meta_specs,
+                                    ),
+                                );
+                            }
+                            None => {
+                                self.retraction_info.push_record(
+                                    RetractionRecord::AddedMetaPredicate(
+                                        module_name.clone(), key,
+                                    )
+                                );
+                            }
+                        }
+                    }
+                    None => {
+                        let module_decl = ModuleDecl {
+                            name: module_name.clone(),
+                            exports: vec![],
+                        };
+
+                        let listing_src = ListingSource::DynamicallyGenerated;
+                        let mut module = Module::new(module_decl, listing_src);
+
+                        module.meta_predicates.insert(key.clone(), meta_specs);
+
+                        self.retraction_info.push_record(
+                            RetractionRecord::AddedMetaPredicate(
+                                module_name.clone(), key,
+                            )
+                        );
+
+                        self.retraction_info.push_record(
+                            RetractionRecord::AddedModule(module_name.clone()),
+                        );
+
+                        self.wam.indices.modules.insert(module_name, module);
+                    }
+                }
+            }
+        }
+    }
+
+    fn import_builtins_in_module(
+        &mut self,
+        code_dir: &mut CodeDir,
+        op_dir: &mut OpDir,
+        meta_predicates: &mut MetaPredicateDir,
+    ) {
+        if let Some(builtins) = self.wam.indices.modules.get(&clause_name!("builtins")) {
+            import_module_exports(
+                &mut self.retraction_info,
+                &self.compilation_target,
+                builtins,
+                code_dir,
+                op_dir,
+                meta_predicates,
+            );
+        }
+    }
+
+    pub(crate)
+    fn add_module(&mut self, module_decl: ModuleDecl, listing_src: ListingSource) {
+        let module_name = module_decl.name.clone();
+
+        let mut module =
+            match self.wam.indices.modules.remove(&module_name) {
+                Some(mut module) => {
+                    let old_module_decl = mem::replace(&mut module.module_decl, module_decl);
+
+                    self.retraction_info.push_record(
+                        RetractionRecord::ReplacedModule(
+                            old_module_decl, listing_src.clone(),
+                        ),
+                    );
+
+                    module.listing_src = listing_src;
+                    module
+                }
+                None => {
+                    self.retraction_info.push_record(
+                        RetractionRecord::AddedModule(module_name.clone()),
+                    );
+
+                    Module::new(module_decl, listing_src)
+                }
+            };
+
+        self.import_builtins_in_module(
+            &mut module.code_dir,
+            &mut module.op_dir,
+            &mut module.meta_predicates,
+        );
+
+        for export in &module.module_decl.exports {
+            if let ModuleExport::OpDecl(ref op_decl) = export {
+                add_op_decl_as_module_export(
+                    &mut module.op_dir,
+                    &self.compilation_target, // this is a Module.
+                    &mut self.retraction_info,
+                    &mut self.wam.indices.op_dir,
+                    &mut self.module_op_exports,
+                    op_decl,
+                );
+            }
+        }
+
+        if let Some(load_context) = self.wam.load_contexts.last_mut() {
+            load_context.module = module_name.clone();
+        }
+
+        self.wam.indices.modules.insert(module_name, module);
+    }
+
+    pub(super)
+    fn import_module(&mut self, module_name: ClauseName) -> Result<(), SessionError> {
+        if let Some(module) = self.wam.indices.modules.remove(&module_name) {
+            match &self.compilation_target {
+                CompilationTarget::User => {
+                    import_module_exports(
+                        &mut self.retraction_info,
+                        &self.compilation_target,
+                        &module,
+                        &mut self.wam.indices.code_dir,
+                        &mut self.wam.indices.op_dir,
+                        &mut self.wam.indices.meta_predicates,
+                    );
+                }
+                CompilationTarget::Module(ref defining_module_name) => {
+                    match self.wam.indices.modules.get_mut(defining_module_name) {
+                        Some(ref mut target_module) => {
+                            import_module_exports_into_module(
+                                &mut self.retraction_info,
+                                &self.compilation_target,
+                                &module,
+                                &mut target_module.code_dir,
+                                &mut target_module.op_dir,
+                                &mut target_module.meta_predicates,
+                                &mut self.wam.indices.op_dir,
+                                &mut self.module_op_exports,
+                            );
+                        }
+                        None => {
+                            // we find ourselves here because we're trying to import
+                            // a module into itself as it is being defined.
+                            self.wam.indices.modules.insert(module_name.clone(), module);
+                            return Err(SessionError::ModuleCannotImportSelf(module_name));
+                        }
+                    }
+                }
+            }
+
+            self.wam.indices.modules.insert(module_name, module);
+            Ok(())
+        } else {
+            Err(SessionError::ExistenceError(ExistenceError::Module(module_name)))
+        }
+    }
+
+    fn import_qualified_module(
+        &mut self,
+        module_name: ClauseName,
+        exports: IndexSet<ModuleExport>,
+    ) -> Result<(), SessionError> {
+        if let Some(module) = self.wam.indices.modules.remove(&module_name) {
+            match &self.compilation_target {
+                CompilationTarget::User => {
+                    import_qualified_module_exports(
+                        &mut self.retraction_info,
+                        &self.compilation_target,
+                        &module,
+                        &exports,
+                        &mut self.wam.indices.code_dir,
+                        &mut self.wam.indices.op_dir,
+                    );
+                }
+                CompilationTarget::Module(ref defining_module_name) => {
+                    match self.wam.indices.modules.get_mut(defining_module_name) {
+                        Some(ref mut target_module) => {
+                            import_qualified_module_exports_into_module(
+                                &mut self.retraction_info,
+                                &self.compilation_target,
+                                &module,
+                                &exports,
+                                &mut target_module.code_dir,
+                                &mut target_module.op_dir,
+                                &mut self.wam.indices.op_dir,
+                                &mut self.module_op_exports,
+                            );
+                        }
+                        None => {
+                            // we find ourselves here because we're trying to import
+                            // a module into itself as it is being defined.
+                            self.wam.indices.modules.insert(module_name.clone(), module);
+                            return Err(SessionError::ModuleCannotImportSelf(module_name));
+                        }
+                    }
+                }
+            }
+
+            self.wam.indices.modules.insert(module_name, module);
+            Ok(())
+        } else {
+            Err(SessionError::ExistenceError(ExistenceError::Module(module_name)))
+        }
+    }
+
+    pub(crate)
+    fn use_module(&mut self, module_src: ModuleSource) -> Result<(), SessionError> {
+        let (stream, listing_src) =
+            match module_src {
+                ModuleSource::File(filename) => {
+                    let mut path_buf = PathBuf::from(filename.as_str());
+                    path_buf.set_extension("pl");
+                    let file = File::open(&path_buf)?;
+
+                    (Stream::from_file_as_input(filename.clone(), file),
+                     ListingSource::File(filename, path_buf))
+                }
+                ModuleSource::Library(library) => {
+                    match LIBRARIES.borrow().get(library.as_str()) {
+                        Some(code) => {
+                            if let Some(ref module) = self.wam.indices.modules.get(&library) {
+                                if let ListingSource::DynamicallyGenerated = &module.listing_src {
+                                    (Stream::from(*code), ListingSource::User)
+                                } else {
+                                    return self.import_module(library);
+                                }
+                            } else {
+                                (Stream::from(*code), ListingSource::User)
+                            }
+                        }
+                        None => {
+                            return self.import_module(library);
+                        }
+                    }
+                }
+            };
+
+        let compilation_target = {
+            let stream = &mut parsing_stream(stream)?;
+
+            let ts = BootstrappingTermStream::from_prolog_stream(
+                stream,
+                self.wam.machine_st.atom_tbl.clone(),
+                self.wam.machine_st.flags,
+                listing_src,
+            );
+
+            let subloader = Loader::new(ts, self.wam);
+            subloader.load()?
+        };
+
+        match compilation_target {
+            CompilationTarget::User => {
+                // nothing to do.
+                Ok(())
+            }
+            CompilationTarget::Module(module_name) => {
+                self.import_module(module_name)
+            }
+        }
+    }
+
+    pub(crate)
+    fn use_qualified_module(
+        &mut self,
+        module_src: ModuleSource,
+        exports: IndexSet<ModuleExport>,
+    ) -> Result<(), SessionError> {
+        let (stream, listing_src) =
+            match module_src {
+                ModuleSource::File(filename) => {
+                    let mut path_buf = PathBuf::from(filename.as_str());
+                    path_buf.set_extension("pl");
+                    let file = File::open(&path_buf)?;
+
+                    (Stream::from_file_as_input(filename.clone(), file),
+                     ListingSource::File(filename, path_buf))
+                }
+                ModuleSource::Library(library) => {
+                    match LIBRARIES.borrow().get(library.as_str()) {
+                        Some(code) => {
+                            if self.wam.indices.modules.contains_key(&library) {
+                                return self.import_qualified_module(library, exports);
+                            } else {
+                                (Stream::from(*code), ListingSource::User)
+                            }
+                        }
+                        None => {
+                            return self.import_qualified_module(library, exports);
+                        }
+                    }
+                }
+            };
+
+        let compilation_target = {
+            let stream = &mut parsing_stream(stream)?;
+
+            let ts = BootstrappingTermStream::from_prolog_stream(
+                stream,
+                self.wam.machine_st.atom_tbl.clone(),
+                self.wam.machine_st.flags,
+                listing_src,
+            );
+
+            let subloader = Loader::new(ts, self.wam);
+            subloader.load()?
+        };
+
+        match compilation_target {
+            CompilationTarget::User => {
+                // nothing to do.
+                Ok(())
+            }
+            CompilationTarget::Module(module_name) => {
+                self.import_qualified_module(module_name, exports)
+            }
+        }
+    }
+
+    #[inline]
+    pub(super)
+    fn composite_op_dir(&self) -> CompositeOpDir {
+        match &self.compilation_target {
+            CompilationTarget::User => {
+                CompositeOpDir::new(&self.wam.indices.op_dir, None)
+            }
+            CompilationTarget::Module(ref module_name) => {
+                match self.wam.indices.modules.get(module_name) {
+                    Some(ref module) => {
+                        CompositeOpDir::new(&self.wam.indices.op_dir, Some(&module.op_dir))
+                    }
+                    None => {
+                        unreachable!()
+                    }
+                }
+            }
+        }
+    }
+}

src/machine/machine_errors.rs

@@ -1,6 +1,6 @@
 use crate::prolog_parser::ast::*;
 
-use crate::forms::{ModuleSource, Number, PredicateKey};
+use crate::forms::{ModuleSource, Number}; //, PredicateKey};
 use crate::machine::heap::*;
 use crate::machine::machine_indices::*;
 use crate::machine::machine_state::*;
@@ -355,24 +355,11 @@ impl MachineError {
         }
     }
 
-    pub(super)
-    fn uninstantiation_error(culprit: Addr) -> Self {
-        let stub = functor!(
-            "uninstantiation_error",
-            [addr(culprit)]
-        );
-
-        MachineError {
-            stub,
-            location: None,
-            from: ErrorProvenance::Received,
-        }
-    }
-
     pub(super)
     fn session_error(h: usize, err: SessionError) -> Self {
         match err {
-            SessionError::CannotOverwriteBuiltIn(pred_str) |
+            // SessionError::CannotOverwriteBuiltIn(pred_str) |
+            /*
             SessionError::CannotOverwriteImport(pred_str) => {
                 Self::permission_error(
                     h,
@@ -381,12 +368,14 @@ impl MachineError {
                     functor!(clause_name(pred_str)),
                 )
             }
+            */
             SessionError::ExistenceError(err) => {
                 Self::existence_error(h, err)
             }
-            SessionError::InvalidFileName(filename) => {
-                Self::existence_error(h, ExistenceError::Module(filename))
-            }
+            // SessionError::InvalidFileName(filename) => {
+            //     Self::existence_error(h, ExistenceError::Module(filename))
+            // }
+            /*
             SessionError::ModuleDoesNotContainExport(..) => {
                 Self::permission_error(
                     h,
@@ -395,6 +384,15 @@ impl MachineError {
                     functor!("module_does_not_contain_claimed_export"),
                 )
             }
+            */
+            SessionError::ModuleCannotImportSelf(module_name) => {
+                Self::permission_error(
+                    h,
+                    Permission::Modify,
+                    "module",
+                    functor!("module_cannot_import_self", [clause_name(module_name)]),
+                )
+            }
             SessionError::NamelessEntry => {
                 Self::permission_error(
                     h,
@@ -411,7 +409,7 @@ impl MachineError {
                     functor!(clause_name(op)),
                 )
             }
-            SessionError::ParserError(err) => {
+            SessionError::CompilationError(err) => {
                 Self::syntax_error(h, err)
             }
             SessionError::QueryCannotBeDefinedAsFact => {
@@ -426,13 +424,15 @@ impl MachineError {
     }
 
     pub(super)
-    fn syntax_error(h: usize, err: ParserError) -> Self {
-        if let ParserError::Arithmetic(err) = err {
+    fn syntax_error<E: Into<CompilationError>>(h: usize, err: E) -> Self {
+        let err = err.into();
+
+        if let CompilationError::Arithmetic(err) = err {
             return Self::arithmetic_error(h, err);
         }
 
         let location = err.line_and_col_num();
-        let stub = functor!(err.as_str());
+        let stub = err.as_functor(h);
 
         let stub = functor!(
             "syntax_error",
@@ -475,9 +475,103 @@ impl MachineError {
     }
 }
 
+#[derive(Debug)]
+pub enum CompilationError {
+    Arithmetic(ArithmeticError),
+    ParserError(ParserError),
+    // BadPendingByte,
+    CannotParseCyclicTerm,
+    // ExpandedTermsListNotAList,
+    ExpectedRel,
+    // ExpectedTopLevelTerm,
+    InadmissibleFact,
+    InadmissibleQueryTerm,
+    InconsistentEntry,
+    // InvalidDoubleQuotesDecl,
+    // InvalidHook,
+    InvalidMetaPredicateDecl,
+    InvalidModuleDecl,
+    InvalidModuleExport,
+    InvalidRuleHead,
+    InvalidUseModuleDecl,
+    InvalidModuleResolution(ClauseName),
+    UnreadableTerm,
+}
+
+impl From<ArithmeticError> for CompilationError {
+    #[inline]
+    fn from(err: ArithmeticError) -> CompilationError {
+        CompilationError::Arithmetic(err)
+    }
+}
+
+impl From<ParserError> for CompilationError {
+    #[inline]
+    fn from(err: ParserError) -> CompilationError {
+        CompilationError::ParserError(err)
+    }
+}
+
+impl CompilationError {
+    pub fn line_and_col_num(&self) -> Option<(usize, usize)> {
+        match self {
+            &CompilationError::ParserError(ref err) =>
+                err.line_and_col_num(),
+            _ =>
+                None
+        }
+    }
+
+    pub fn as_functor(&self, _h: usize) -> MachineStub {
+        match self {
+            &CompilationError::Arithmetic(..) =>
+                functor!("arithmetic_error"),
+            // &CompilationError::BadPendingByte =>
+            //     functor!("bad_pending_byte"),
+            &CompilationError::CannotParseCyclicTerm =>
+                functor!("cannot_parse_cyclic_term"),
+            // &CompilationError::ExpandedTermsListNotAList =>
+            //     functor!("expanded_terms_list_is_not_a_list"),
+            &CompilationError::ExpectedRel =>
+                functor!("expected_relation"),
+            // &CompilationError::ExpectedTopLevelTerm =>
+            //     functor!("expected_atom_or_cons_or_clause"),
+            &CompilationError::InadmissibleFact =>
+                functor!("inadmissible_fact"),
+            &CompilationError::InadmissibleQueryTerm =>
+                functor!("inadmissible_query_term"),
+            &CompilationError::InconsistentEntry =>
+                functor!("inconsistent_entry"),
+            // &CompilationError::InvalidDoubleQuotesDecl =>
+            //     functor!("invalid_double_quotes_declaration"),
+            // &CompilationError::InvalidHook =>
+            //     functor!("invalid_hook"),
+            &CompilationError::InvalidMetaPredicateDecl =>
+                functor!("invalid_meta_predicate_decl"),
+            &CompilationError::InvalidModuleDecl =>
+                functor!("invalid_module_declaration"),
+            &CompilationError::InvalidModuleExport =>
+                functor!("invalid_module_export"),
+            &CompilationError::InvalidModuleResolution(ref module_name) =>
+                functor!(
+                    "no_such_module",
+                    [clause_name(module_name.clone())]
+                ),
+            &CompilationError::InvalidRuleHead =>
+                functor!("invalid_head_of_rule"),
+            &CompilationError::InvalidUseModuleDecl =>
+                functor!("invalid_use_module_declaration"),
+            &CompilationError::ParserError(ref err) =>
+                functor!(err.as_str()),
+            &CompilationError::UnreadableTerm =>
+                functor!("unreadable_term"),
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub enum Permission {
-    Access,
+    // Access,
     Create,
     InputStream,
     Modify,
@@ -490,7 +584,7 @@ impl Permission {
     #[inline]
     pub fn as_str(self) -> &'static str {
         match self {
-            Permission::Access => "access",
+            // Permission::Access => "access",
             Permission::Create => "create",
             Permission::InputStream => "input",
             Permission::Modify => "modify",
@@ -807,37 +901,55 @@ pub enum ExistenceError {
 
 #[derive(Debug)]
 pub enum SessionError {
-    CannotOverwriteBuiltIn(ClauseName),
-    CannotOverwriteImport(ClauseName),
+    CompilationError(CompilationError),
+    // CannotOverwriteBuiltIn(ClauseName),
+    // CannotOverwriteImport(ClauseName),
     ExistenceError(ExistenceError),
-    InvalidFileName(ClauseName),
-    ModuleDoesNotContainExport(ClauseName, PredicateKey),
+    // InvalidFileName(ClauseName),
+    // ModuleDoesNotContainExport(ClauseName, PredicateKey),
+    ModuleCannotImportSelf(ClauseName),
     NamelessEntry,
     OpIsInfixAndPostFix(ClauseName),
     QueryCannotBeDefinedAsFact,
-    ParserError(ParserError),
 }
 
 #[derive(Debug)]
 pub enum EvalSession {
-    EntrySuccess,
+    // EntrySuccess,
     Error(SessionError),
 }
 
 impl From<SessionError> for EvalSession {
+    #[inline]
     fn from(err: SessionError) -> Self {
         EvalSession::Error(err)
     }
 }
 
+impl From<std::io::Error> for SessionError {
+    #[inline]
+    fn from(err: std::io::Error) -> SessionError {
+        SessionError::from(ParserError::from(err))
+    }
+}
+
 impl From<ParserError> for SessionError {
+    #[inline]
     fn from(err: ParserError) -> Self {
-        SessionError::ParserError(err)
+        SessionError::CompilationError(CompilationError::from(err))
+    }
+}
+
+impl From<CompilationError> for SessionError {
+    #[inline]
+    fn from(err: CompilationError) -> Self {
+        SessionError::CompilationError(err)
     }
 }
 
 impl From<ParserError> for EvalSession {
+    #[inline]
     fn from(err: ParserError) -> Self {
-        EvalSession::from(SessionError::ParserError(err))
+        EvalSession::from(SessionError::from(err))
     }
 }

src/machine/machine_indices.rs

@@ -1,9 +1,9 @@
 use crate::prolog_parser::ast::*;
-use crate::prolog_parser::tabled_rc::*;
 
 use crate::clause_types::*;
 use crate::fixtures::*;
 use crate::forms::*;
+use crate::machine::CompilationTarget;
 use crate::machine::code_repo::CodeRepo;
 use crate::machine::Ball;
 use crate::machine::heap::*;
@@ -11,20 +11,21 @@ use crate::machine::machine_state::*;
 use crate::machine::partial_string::*;
 use crate::machine::raw_block::RawBlockTraits;
 use crate::machine::streams::Stream;
+use crate::machine::term_stream::LoadStatePayload;
 use crate::instructions::*;
 use crate::ordered_float::OrderedFloat;
 use crate::rug::{Integer, Rational};
 
 use crate::indexmap::IndexMap;
 
-use std::cell::RefCell;
+use std::cell::Cell;
 use std::cmp::Ordering;
-use std::collections::{BTreeMap, BTreeSet, VecDeque};
+use std::collections::{BTreeMap, BTreeSet};
 use std::convert::TryFrom;
 use std::fmt;
-use std::mem;
+// use std::mem;
 use std::net::TcpListener;
-use std::ops::{Add, AddAssign, Sub, SubAssign};
+use std::ops::{Add, AddAssign, Deref, Sub, SubAssign};
 use std::rc::Rc;
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -64,6 +65,7 @@ pub enum Addr {
     Fixnum(isize),
     Float(OrderedFloat<f64>),
     Lis(usize),
+    LoadStatePayload(usize),
     HeapCell(usize),
     PStrLocation(usize, usize), // location of pstr in heap, offset into string in bytes.
     StackCell(usize, usize),
@@ -231,7 +233,7 @@ impl Addr {
                     Addr::Lis(_) | Addr::PStrLocation(..) | Addr::Str(_) => {
                         Some(TermOrderCategory::Compound)
                     }
-                    Addr::CutPoint(_) | Addr::Stream(_) | Addr::TcpListener(_) => {
+                    Addr::CutPoint(_) | Addr::LoadStatePayload(_) | Addr::Stream(_) | Addr::TcpListener(_) => {
                         None
                     }
                 }
@@ -369,6 +371,7 @@ pub enum HeapCellValue {
     Atom(ClauseName, Option<SharedOpDesc>),
     DBRef(DBRef),
     Integer(Rc<Integer>),
+    LoadStatePayload(LoadStatePayload),
     NamedStr(usize, ClauseName, Option<SharedOpDesc>), // arity, name, precedence/Specifier if it has one.
     Rational(Rc<Rational>),
     PartialString(PartialString, bool), // the partial string, a bool indicating whether it came from a Constant.
@@ -387,6 +390,9 @@ impl HeapCellValue {
             HeapCellValue::Rational(..) => {
                 Addr::Con(focus)
             }
+            HeapCellValue::LoadStatePayload(_) => {
+                Addr::LoadStatePayload(focus)
+            }
             HeapCellValue::NamedStr(_, _, _) => {
                 Addr::Str(focus)
             }
@@ -417,6 +423,9 @@ impl HeapCellValue {
             &HeapCellValue::Integer(ref n) => {
                 HeapCellValue::Integer(n.clone())
             }
+            &HeapCellValue::LoadStatePayload(_) => {
+                HeapCellValue::Atom(clause_name!("$live_term_stream"), None)
+            }
             &HeapCellValue::NamedStr(arity, ref name, ref op) => {
                 HeapCellValue::NamedStr(arity, name.clone(), op.clone())
             }
@@ -443,50 +452,42 @@ impl From<Addr> for HeapCellValue {
     }
 }
 
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(Debug, Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)]
 pub enum IndexPtr {
     DynamicUndefined, // a predicate, declared as dynamic, whose location in code is as yet undefined.
-    Undefined,
-    InSituDirEntry(usize),
     Index(usize),
-    UserGoalExpansion,
-    UserTermExpansion
+    Undefined,
 }
 
 #[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)]
-pub struct CodeIndex(pub Rc<RefCell<(IndexPtr, ClauseName)>>);
+pub struct CodeIndex(pub Rc<Cell<IndexPtr>>);
+
+impl Deref for CodeIndex {
+    type Target = Cell<IndexPtr>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        self.0.deref()
+    }
+}
 
 impl CodeIndex {
     #[inline]
-    pub fn new(ptr: IndexPtr, module_name: ClauseName) -> Self {
-        CodeIndex(Rc::new(RefCell::new(( ptr, module_name ))))
+    pub(super)
+    fn new(ptr: IndexPtr) -> Self {
+        CodeIndex(Rc::new(Cell::new(ptr)))
     }
 
     #[inline]
     pub fn is_undefined(&self) -> bool {
-        let index_ptr = &self.0.borrow().0;
-
-        match index_ptr {
-            &IndexPtr::Undefined | &IndexPtr::DynamicUndefined => true,
+        match self.0.get() {
+            IndexPtr::Undefined => true, // | &IndexPtr::DynamicUndefined => true,
             _ => false
         }
     }
 
-    #[inline]
-    pub fn dynamic_undefined(module_name: ClauseName) -> Self {
-        CodeIndex(Rc::new(RefCell::new((
-            IndexPtr::DynamicUndefined,
-            module_name
-        ))))
-    }
-
-    #[inline]
-    pub fn module_name(&self) -> ClauseName {
-        self.0.borrow().1.clone()
-    }
-
     pub fn local(&self) -> Option<usize> {
-        match self.0.borrow().0 {
+        match self.0.get() {
             IndexPtr::Index(i) => Some(i),
             _ => None,
         }
@@ -495,60 +496,34 @@ impl CodeIndex {
 
 impl Default for CodeIndex {
     fn default() -> Self {
-        CodeIndex(Rc::new(RefCell::new((
-            IndexPtr::Undefined,
-            clause_name!(""),
-        ))))
+        CodeIndex(Rc::new(Cell::new(IndexPtr::Undefined)))
     }
 }
 
-impl From<(usize, ClauseName)> for CodeIndex {
-    fn from(value: (usize, ClauseName)) -> Self {
-        CodeIndex(Rc::new(RefCell::new((IndexPtr::Index(value.0), value.1))))
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum DynamicAssertPlace {
-    Back,
-    Front,
-}
-
-impl DynamicAssertPlace {
-    #[inline]
-    pub fn predicate_name(self) -> ClauseName {
-        match self {
-            DynamicAssertPlace::Back => clause_name!("assertz"),
-            DynamicAssertPlace::Front => clause_name!("asserta"),
-        }
-    }
-
-    #[inline]
-    pub fn push_to_queue(self, addrs: &mut VecDeque<Addr>, new_addr: Addr) {
-        match self {
-            DynamicAssertPlace::Back => addrs.push_back(new_addr),
-            DynamicAssertPlace::Front => addrs.push_front(new_addr),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum DynamicTransactionType {
-    Abolish,
-    Assert(DynamicAssertPlace),
-    ModuleAbolish,
-    ModuleAssert(DynamicAssertPlace),
-    ModuleRetract,
-    Retract, // dynamic index of the clause to remove.
-}
-
 #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)]
 pub enum REPLCodePtr {
-    CompileBatch,
+    AddDynamicPredicate,
+    AddGoalExpansionClause,
+    AddTermExpansionClause,
+    ClauseToEvacuable,
+    ConcludeLoad,
+    DeclareModule,
+    LoadCompiledLibrary,
+    LoadContextSource,
+    LoadContextFile,
+    LoadContextDirectory,
+    LoadContextModule,
+    LoadContextStream,
+    PopLoadContext,
+    PopLoadStatePayload,
+    PushLoadContext,
+    PushLoadStatePayload,
     UseModule,
-    UseQualifiedModule,
-    UseModuleFromFile,
-    UseQualifiedModuleFromFile
+    MetaPredicateProperty,
+    CompilePendingPredicates,
+    UserAsserta,
+    UserAssertz,
+    UserRetract,
 }
 
 #[derive(Debug, Clone, PartialEq)]
@@ -556,7 +531,7 @@ pub enum CodePtr {
     BuiltInClause(BuiltInClauseType, LocalCodePtr), // local is the successor call.
     CallN(usize, LocalCodePtr, bool),               // arity, local, last call.
     Local(LocalCodePtr),
-    DynamicTransaction(DynamicTransactionType, LocalCodePtr), // the type of transaction, the return pointer.
+    // DynamicTransaction(DynamicTransactionType, LocalCodePtr), // the type of transaction, the return pointer.
     REPL(REPLCodePtr, LocalCodePtr),                          // the REPL code, the return pointer.
     VerifyAttrInterrupt(usize), // location of the verify attribute interrupt code in the CodeDir.
 }
@@ -568,18 +543,26 @@ impl CodePtr {
           | &CodePtr::CallN(_, ref local, _)
           | &CodePtr::Local(ref local) => local.clone(),
             &CodePtr::VerifyAttrInterrupt(p) => LocalCodePtr::DirEntry(p),
-            &CodePtr::REPL(_, p) | &CodePtr::DynamicTransaction(_, p) => p,
+            &CodePtr::REPL(_, p) => p // | &CodePtr::DynamicTransaction(_, p) => p,
+        }
+    }
+
+    #[inline]
+    pub fn is_halt(&self) -> bool {
+        if let CodePtr::Local(LocalCodePtr::Halt) = self {
+            true
+        } else {
+            false
         }
     }
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
 pub enum LocalCodePtr {
-    DirEntry(usize), // offset.
-    InSituDirEntry(usize),
-    TopLevel(usize, usize), // chunk_num, offset.
-    UserGoalExpansion(usize),
-    UserTermExpansion(usize),
+    DirEntry(usize), // offset
+    Halt,
+    IndexingBuf(usize, usize, usize), // DirEntry offset, first internal offset, second internal offset
+    // TopLevel(usize, usize), // chunk_num, offset
 }
 
 impl LocalCodePtr {
@@ -591,6 +574,16 @@ impl LocalCodePtr {
         }
     }
 
+    #[inline]
+    pub(crate)
+    fn abs_loc(&self) -> usize {
+        match self {
+            LocalCodePtr::DirEntry(ref p) => *p,
+            LocalCodePtr::IndexingBuf(ref p, ..) => *p,
+            LocalCodePtr::Halt => unreachable!(),
+        }
+    }
+
     pub(crate)
     fn is_reset_cont_marker(&self, code_repo: &CodeRepo, last_call: bool) -> bool {
         match code_repo.lookup_instr(last_call, &CodePtr::Local(*self)) {
@@ -621,28 +614,21 @@ impl LocalCodePtr {
                     [integer(*p)]
                 ));
             }
-            LocalCodePtr::InSituDirEntry(p) => {
-                heap.append(functor!(
-                    "in_situ_dir_entry",
-                    [integer(*p)]
-                ));
+            LocalCodePtr::Halt => {
+                heap.append(functor!("halt"));
             }
+            /*
             LocalCodePtr::TopLevel(chunk_num, offset) => {
                 heap.append(functor!(
                     "top_level",
                     [integer(*chunk_num), integer(*offset)]
                 ));
             }
-            LocalCodePtr::UserGoalExpansion(p) => {
+            */
+            LocalCodePtr::IndexingBuf(p, o, i) => {
                 heap.append(functor!(
-                    "user_goal_expansion",
-                    [integer(*p)]
-                ));
-            }
-            LocalCodePtr::UserTermExpansion(p) => {
-                heap.append(functor!(
-                    "user_term_expansion",
-                    [integer(*p)]
+                    "indexed_buf",
+                    [integer(*p), integer(*o), integer(*i)]
                 ));
             }
         }
@@ -651,6 +637,7 @@ impl LocalCodePtr {
     }
 }
 
+/*
 impl PartialOrd<CodePtr> for CodePtr {
     fn partial_cmp(&self, other: &CodePtr) -> Option<Ordering> {
         match (self, other) {
@@ -667,11 +654,8 @@ impl PartialOrd<CodePtr> for CodePtr {
 impl PartialOrd<LocalCodePtr> for LocalCodePtr {
     fn partial_cmp(&self, other: &LocalCodePtr) -> Option<Ordering> {
         match (self, other) {
-            (&LocalCodePtr::InSituDirEntry(p1), &LocalCodePtr::InSituDirEntry(ref p2))
-	  | (&LocalCodePtr::DirEntry(p1), &LocalCodePtr::DirEntry(ref p2))
-          | (&LocalCodePtr::UserTermExpansion(p1), &LocalCodePtr::UserTermExpansion(ref p2))
-          | (&LocalCodePtr::UserGoalExpansion(p1), &LocalCodePtr::UserGoalExpansion(ref p2))
-          | (&LocalCodePtr::TopLevel(_, p1), &LocalCodePtr::TopLevel(_, ref p2)) => {
+	        (&LocalCodePtr::DirEntry(p1), &LocalCodePtr::DirEntry(ref p2)) |
+            (&LocalCodePtr::TopLevel(_, p1), &LocalCodePtr::TopLevel(_, ref p2)) => {
                 p1.partial_cmp(p2)
             }
             (_, &LocalCodePtr::TopLevel(_, _)) => {
@@ -683,29 +667,34 @@ impl PartialOrd<LocalCodePtr> for LocalCodePtr {
         }
     }
 }
+*/
 
 impl Default for CodePtr {
+    #[inline]
     fn default() -> Self {
         CodePtr::Local(LocalCodePtr::default())
     }
 }
 
 impl Default for LocalCodePtr {
+    #[inline]
     fn default() -> Self {
-        LocalCodePtr::TopLevel(0, 0)
+        LocalCodePtr::DirEntry(0)
     }
 }
 
 impl Add<usize> for LocalCodePtr {
     type Output = LocalCodePtr;
 
+    #[inline]
     fn add(self, rhs: usize) -> Self::Output {
         match self {
-            LocalCodePtr::InSituDirEntry(p) => LocalCodePtr::InSituDirEntry(p + rhs),
-            LocalCodePtr::DirEntry(p) => LocalCodePtr::DirEntry(p + rhs),
-            LocalCodePtr::TopLevel(cn, p) => LocalCodePtr::TopLevel(cn, p + rhs),
-            LocalCodePtr::UserTermExpansion(p) => LocalCodePtr::UserTermExpansion(p + rhs),
-            LocalCodePtr::UserGoalExpansion(p) => LocalCodePtr::UserGoalExpansion(p + rhs),
+            LocalCodePtr::DirEntry(p) =>
+                LocalCodePtr::DirEntry(p + rhs),
+            LocalCodePtr::Halt =>
+                unreachable!(),
+            LocalCodePtr::IndexingBuf(p, o, i) =>
+                LocalCodePtr::IndexingBuf(p, o, i + rhs),
         }
     }
 }
@@ -713,30 +702,40 @@ impl Add<usize> for LocalCodePtr {
 impl Sub<usize> for LocalCodePtr {
     type Output = Option<LocalCodePtr>;
 
+    #[inline]
     fn sub(self, rhs: usize) -> Self::Output {
         match self {
-            LocalCodePtr::InSituDirEntry(p) =>
-                p.checked_sub(rhs).map(LocalCodePtr::InSituDirEntry),
             LocalCodePtr::DirEntry(p) =>
                 p.checked_sub(rhs).map(LocalCodePtr::DirEntry),
-            LocalCodePtr::TopLevel(cn, p) =>
-                p.checked_sub(rhs).map(|r| LocalCodePtr::TopLevel(cn, r)),
-            LocalCodePtr::UserTermExpansion(p) =>
-                p.checked_sub(rhs).map(LocalCodePtr::UserTermExpansion),
-            LocalCodePtr::UserGoalExpansion(p) =>
-                p.checked_sub(rhs).map(LocalCodePtr::UserGoalExpansion),
+            LocalCodePtr::Halt =>
+                unreachable!(),
+            LocalCodePtr::IndexingBuf(p, o, i) =>
+                i.checked_sub(rhs).map(|r| LocalCodePtr::IndexingBuf(p, o, r)),
         }
     }
 }
 
+impl SubAssign<usize> for LocalCodePtr {
+    #[inline]
+    fn sub_assign(&mut self, rhs: usize) {
+        match self {
+            LocalCodePtr::DirEntry(ref mut p) =>
+                *p -= rhs,
+            LocalCodePtr::Halt | LocalCodePtr::IndexingBuf(..) =>
+                unreachable!(),
+        }
+    }
+}
+
+
 impl AddAssign<usize> for LocalCodePtr {
+    #[inline]
     fn add_assign(&mut self, rhs: usize) {
         match self {
-            &mut LocalCodePtr::InSituDirEntry(ref mut p)
-          | &mut LocalCodePtr::UserGoalExpansion(ref mut p)
-          | &mut LocalCodePtr::UserTermExpansion(ref mut p)
-          | &mut LocalCodePtr::DirEntry(ref mut p)
-          | &mut LocalCodePtr::TopLevel(_, ref mut p) => *p += rhs,
+            &mut LocalCodePtr::DirEntry(ref mut p) /* |
+            &mut LocalCodePtr::TopLevel(_, ref mut p) */    => *p += rhs,
+            &mut LocalCodePtr::IndexingBuf(_, _, ref mut i) => *i += rhs,
+            &mut LocalCodePtr::Halt => unreachable!(),
         }
     }
 }
@@ -746,10 +745,14 @@ impl Add<usize> for CodePtr {
 
     fn add(self, rhs: usize) -> Self::Output {
         match self {
-            p @ CodePtr::REPL(..)
-          | p @ CodePtr::VerifyAttrInterrupt(_)
-          | p @ CodePtr::DynamicTransaction(..) => p,
-            CodePtr::Local(local) => CodePtr::Local(local + rhs),
+            p @ CodePtr::REPL(..) |
+            p @ CodePtr::VerifyAttrInterrupt(_) => { // |
+            // p @ CodePtr::DynamicTransaction(..) => {
+                p
+            }
+            CodePtr::Local(local) => {
+                CodePtr::Local(local + rhs)
+            }
             CodePtr::BuiltInClause(_, local) | CodePtr::CallN(_, local, _) => {
                 CodePtr::Local(local + rhs)
             }
@@ -767,186 +770,206 @@ impl AddAssign<usize> for CodePtr {
     }
 }
 
-pub type HeapVarDict = IndexMap<Rc<Var>, Addr>;
-pub type AllocVarDict = IndexMap<Rc<Var>, VarData>;
-
-#[derive(Debug, Clone)]
-pub struct DynamicPredicateInfo {
-    pub(super) clauses_subsection_p: usize, // a LocalCodePtr::DirEntry value.
-}
-
-impl Default for DynamicPredicateInfo {
-    fn default() -> Self {
-        DynamicPredicateInfo {
-            clauses_subsection_p: 0,
+impl SubAssign<usize> for CodePtr {
+    #[inline]
+    fn sub_assign(&mut self, rhs: usize) {
+        match self {
+            CodePtr::Local(ref mut local) => *local -= rhs,
+            _ => unreachable!(),
         }
     }
 }
 
+
+pub type HeapVarDict = IndexMap<Rc<Var>, Addr>;
+pub type AllocVarDict = IndexMap<Rc<Var>, VarData>;
+
 pub type InSituCodeDir = IndexMap<PredicateKey, usize>;
-
-// key type: module name, predicate indicator.
-pub type DynamicCodeDir = IndexMap<(ClauseName, ClauseName, usize), DynamicPredicateInfo>;
-
 pub type GlobalVarDir = IndexMap<ClauseName, (Ball, Option<usize>)>;
 
 #[derive(Debug)]
 pub(crate) struct ModuleStub {
-    pub(crate) atom_tbl: TabledData<Atom>,
     pub(crate) in_situ_code_dir: InSituCodeDir,
 }
 
-impl ModuleStub {
-    pub(crate) fn new(atom_tbl: TabledData<Atom>) -> Self {
-        ModuleStub {
-            atom_tbl,
-            in_situ_code_dir: InSituCodeDir::new(),
-        }
-    }
-}
-
-pub(crate) type ModuleStubDir = IndexMap<ClauseName, ModuleStub>;
+// pub(crate) type ModuleStubDir = IndexMap<ClauseName, ModuleStub>;
 pub(crate) type StreamAliasDir = IndexMap<ClauseName, Stream>;
 pub(crate) type StreamDir = BTreeSet<Stream>;
 
+pub type MetaPredicateDir = IndexMap<PredicateKey, Vec<MetaSpec>>;
+
+pub type ExtensiblePredicates = IndexMap<PredicateKey, PredicateSkeleton>;
+
 #[derive(Debug)]
 pub struct IndexStore {
-    pub(super) atom_tbl: TabledData<Atom>,
     pub(super) code_dir: CodeDir,
-    pub(super) dynamic_code_dir: DynamicCodeDir,
+    pub(super) extensible_predicates: ExtensiblePredicates,
     pub(super) global_variables: GlobalVarDir,
-    pub(super) in_situ_code_dir: InSituCodeDir,
-    pub(super) in_situ_module_dir: ModuleStubDir,
-    pub(super) module_dir: ModuleDir,
+    pub(super) meta_predicates: MetaPredicateDir,
     pub(super) modules: ModuleDir,
     pub(super) op_dir: OpDir,
     pub(super) streams: StreamDir,
     pub(super) stream_aliases: StreamAliasDir,
 }
 
+impl Default for IndexStore {
+    #[inline]
+    fn default() -> Self {
+        index_store!(CodeDir::new(), default_op_dir(), ModuleDir::new())
+    }
+}
+
 impl IndexStore {
-    pub fn predicate_exists(
-        &self,
-        name: ClauseName,
-        module: ClauseName,
-        arity: usize,
-        op_spec: Option<SharedOpDesc>,
-    ) -> bool {
-        match self.modules.get(&module) {
-            Some(module) => match ClauseType::from(name, arity, op_spec) {
-                ClauseType::Named(name, arity, _) => module.code_dir.contains_key(&(name, arity)),
-                ClauseType::Op(name, spec, ..) => {
-                    module.code_dir.contains_key(&(name, spec.arity()))
+    pub fn get_predicate_skeleton(
+        &mut self,
+        compilation_target: &CompilationTarget,
+        key: &PredicateKey,
+    ) -> Option<&mut PredicateSkeleton> {
+        match (key.0.as_str(), key.1) {
+            ("term_expansion", 2) => {
+                self.extensible_predicates.get_mut(key)
+            }
+            _ => {
+                match compilation_target {
+                    CompilationTarget::User => {
+                        self.extensible_predicates.get_mut(key)
+                    }
+                    CompilationTarget::Module(ref module_name) => {
+                        if let Some(module) = self.modules.get_mut(module_name) {
+                            module.extensible_predicates.get_mut(key)
+                        } else {
+                            None
+                        }
+                    }
                 }
-                _ => true,
-            },
-            None => match ClauseType::from(name, arity, op_spec) {
-                ClauseType::Named(name, arity, _) => self.code_dir.contains_key(&(name, arity)),
-                ClauseType::Op(name, spec, ..) => self.code_dir.contains_key(&(name, spec.arity())),
-                _ => true,
-            },
+            }
         }
     }
 
-    pub fn add_term_and_goal_expansion_indices(&mut self) {
-        self.code_dir.insert((clause_name!("term_expansion"), 2),
-                             CodeIndex(Rc::new(RefCell::new(
-                                 (IndexPtr::UserTermExpansion,
-                                  clause_name!("user"))
-                             ))));
-        self.code_dir.insert((clause_name!("goal_expansion"), 2),
-                             CodeIndex(Rc::new(RefCell::new(
-                                 (IndexPtr::UserGoalExpansion,
-                                  clause_name!("user"))
-                             ))));
+    pub fn remove_predicate_skeleton(
+        &mut self,
+        compilation_target: &CompilationTarget,
+        key: &PredicateKey,
+    ) {
+        match (key.0.as_str(), key.1) {
+            ("term_expansion", 2) => {
+                self.extensible_predicates.remove(key);
+            },
+            _ => {
+                match compilation_target {
+                    CompilationTarget::User => {
+                        self.extensible_predicates.remove(key);
+                    }
+                    CompilationTarget::Module(ref module_name) => {
+                        if let Some(module) = self.modules.get_mut(module_name) {
+                            module.extensible_predicates.remove(key);
+                        }
+                    }
+                }
+            }
+        }
     }
 
-    #[inline]
-    pub fn remove_clause_subsection(&mut self, module: ClauseName, name: ClauseName, arity: usize) {
-        self.dynamic_code_dir.swap_remove(&(module, name, arity));
-    }
-
-    #[inline]
-    pub fn get_clause_subsection(
+    pub fn get_predicate_code_index(
         &self,
-        module: ClauseName,
         name: ClauseName,
         arity: usize,
-    ) -> Option<DynamicPredicateInfo> {
-        self.dynamic_code_dir.get(&(module, name, arity)).cloned()
+        module: ClauseName,
+        op_spec: Option<SharedOpDesc>,
+    ) -> Option<CodeIndex> {
+        if module.as_str() == "user" {
+            match ClauseType::from(name, arity, op_spec) {
+                ClauseType::Named(name, arity, _) => {
+                    self.code_dir.get(&(name, arity)).cloned()
+                }
+                ClauseType::Op(name, spec, ..) => {
+                    self.code_dir.get(&(name, spec.arity())).cloned()
+                }
+                _ => {
+                    None
+                }
+            }
+        } else {
+            self.modules.get(&module).and_then(|module| {
+                match ClauseType::from(name, arity, op_spec) {
+                    ClauseType::Named(name, arity, _) => {
+                        module.code_dir.get(&(name, arity)).cloned()
+                    }
+                    ClauseType::Op(name, spec, ..) => {
+                        module.code_dir.get(&(name, spec.arity())).cloned()
+                    }
+                    _ => {
+                        None
+                    }
+                }
+            })
+        }
     }
 
-    #[inline]
-    pub(crate) fn take_in_situ_module_dir(&mut self) -> ModuleStubDir {
-        mem::replace(&mut self.in_situ_module_dir, ModuleStubDir::new())
+    pub fn get_meta_predicate_spec(
+        &self,
+        name: ClauseName,
+        arity: usize,
+        compilation_target: &CompilationTarget,
+    ) -> Option<&Vec<MetaSpec>> {
+        match compilation_target {
+            CompilationTarget::User => {
+                self.meta_predicates.get(&(name, arity))
+            }
+            CompilationTarget::Module(ref module_name) => {
+                match self.modules.get(module_name) {
+                    Some(ref module) => {
+                        module.meta_predicates.get(&(name.clone(), arity))
+                              .or_else(|| {
+                                  self.meta_predicates.get(&(name, arity))
+                              })
+                    }
+                    None => {
+                        self.meta_predicates.get(&(name, arity))
+                    }
+                }
+            }
+        }
     }
 
-    #[inline]
-    pub fn take_in_situ_code_dir(&mut self) -> InSituCodeDir {
-        mem::replace(&mut self.in_situ_code_dir, InSituCodeDir::new())
-    }
-
-    #[inline]
-    pub fn take_module(&mut self, name: ClauseName) -> Option<Module> {
-        self.modules.swap_remove(&name)
-    }
-
-    #[inline]
-    pub fn insert_module(&mut self, module: Module) {
-        self.modules.insert(module.module_decl.name.clone(), module);
+    pub fn is_dynamic_predicate(&self, module_name: ClauseName, key: PredicateKey) -> bool {
+        match module_name.as_str() {
+            "user" => {
+                self.extensible_predicates.get(&key)
+                    .map(|skeleton| skeleton.is_dynamic)
+                    .unwrap_or(false)
+            }
+            _ => {
+                match self.modules.get(&module_name) {
+                    Some(ref module) => {
+                        module.extensible_predicates.get(&key)
+                              .map(|skeleton| skeleton.is_dynamic)
+                              .unwrap_or(false)
+                    }
+                    None => {
+                        false
+                    }
+                }
+            }
+        }
     }
 
     #[inline]
     pub(super) fn new() -> Self {
-        IndexStore {
-            atom_tbl: TabledData::new(Rc::new("user".to_string())),
-            code_dir: CodeDir::new(),
-            module_dir: ModuleDir::new(),
-            dynamic_code_dir: DynamicCodeDir::new(),
-            global_variables: GlobalVarDir::new(),
-            in_situ_code_dir: InSituCodeDir::new(),
-            in_situ_module_dir: ModuleStubDir::new(),
-            op_dir: default_op_dir(),
-            modules: ModuleDir::new(),
-            stream_aliases: StreamAliasDir::new(),
-            streams: StreamDir::new(),
-        }
+        IndexStore::default()
     }
 
-    #[inline]
-    pub(super) fn copy_and_swap(&mut self, other: &mut IndexStore) {
-        self.code_dir = other.code_dir.clone();
-        self.op_dir = other.op_dir.clone();
-
-        mem::swap(&mut self.code_dir, &mut other.code_dir);
-        mem::swap(&mut self.op_dir, &mut other.op_dir);
-        mem::swap(&mut self.modules, &mut other.modules);
-    }
-
-    #[inline]
-    fn get_internal(
-        &self,
-        name: ClauseName,
-        arity: usize,
-        in_mod: ClauseName,
-    ) -> Option<CodeIndex> {
-        self.modules
-            .get(&in_mod)
-            .and_then(|ref module| module.code_dir.get(&(name, arity)))
-            .cloned()
-    }
-
-    pub(super) fn get_cleaner_sites(&self) -> (usize, usize) {
+    pub(super)
+    fn get_cleaner_sites(&self) -> (usize, usize) {
         let r_w_h = clause_name!("run_cleaners_with_handling");
         let r_wo_h = clause_name!("run_cleaners_without_handling");
-
         let iso_ext = clause_name!("iso_ext");
 
         let r_w_h = self
-            .get_internal(r_w_h, 0, iso_ext.clone())
+            .get_predicate_code_index(r_w_h, 0, iso_ext.clone(), None)
             .and_then(|item| item.local());
         let r_wo_h = self
-            .get_internal(r_wo_h, 1, iso_ext)
+            .get_predicate_code_index(r_wo_h, 1, iso_ext, None)
             .and_then(|item| item.local());
 
         if let Some(r_w_h) = r_w_h {
@@ -960,129 +983,6 @@ impl IndexStore {
 }
 
 pub type CodeDir = BTreeMap<PredicateKey, CodeIndex>;
-pub type TermDir = IndexMap<PredicateKey, (Predicate, VecDeque<TopLevel>)>;
-
-#[derive(Debug)]
-pub struct TermDirQuantumEntry {
-    pub old_terms: (Predicate, VecDeque<TopLevel>),
-    pub new_terms: (Predicate, VecDeque<TopLevel>),
-    pub is_fresh: bool,
-}
-
-impl TermDirQuantumEntry {
-    #[inline]
-    pub fn new() -> Self {
-        TermDirQuantumEntry {
-            old_terms: (Predicate::new(), VecDeque::new()),
-            new_terms: (Predicate::new(), VecDeque::new()),
-            is_fresh: false,
-        }
-    }
-
-    pub fn from(preds: &Predicate, queue: &VecDeque<TopLevel>) -> Self
-    {
-        let mut entry = TermDirQuantumEntry::new();
-        entry.is_fresh = false;
-
-        (entry.old_terms.0).0.extend(preds.0.iter().cloned());
-        entry.old_terms.1.extend(queue.iter().cloned());
-
-        entry
-    }
-}
-
-#[derive(Debug)]
-pub struct TermDirQuantum(IndexMap<PredicateKey, TermDirQuantumEntry>);
-
-impl TermDirQuantum {
-    #[inline]
-    pub fn new() -> Self {
-        TermDirQuantum(IndexMap::new())
-    }
-
-    #[inline]
-    pub fn insert_or_refresh(&mut self, key: PredicateKey, mut entry: TermDirQuantumEntry) {
-        if let Some(prev_entry) = self.get_mut(&key) {
-            prev_entry.is_fresh = true;
-        } else {
-            entry.is_fresh = true;
-            self.0.insert(key, entry);
-        }
-    }
-
-    #[inline]
-    pub fn insert(&mut self, key: PredicateKey, entry: TermDirQuantumEntry) {
-        self.0.insert(key, entry);
-    }
-
-    #[inline]
-    pub fn get_mut(&mut self, key: &PredicateKey) -> Option<&mut TermDirQuantumEntry> {
-        self.0.get_mut(key)
-    }
-
-    pub fn consolidate(self) -> TermDir {
-        let mut term_dir = TermDir::new();
-
-        for (key, entry) in self.0 {
-            let (preds, queue) =
-                term_dir.entry(key).or_insert((Predicate::new(), VecDeque::new()));
-
-            preds.0.extend((entry.new_terms.0).0.into_iter());
-            queue.extend(entry.new_terms.1.into_iter());
-        }
-
-        term_dir
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
-pub enum CompileTimeHook {
-    GoalExpansion,
-    TermExpansion,
-    UserGoalExpansion,
-    UserTermExpansion,
-}
-
-impl CompileTimeHook {
-    pub fn name(self) -> ClauseName {
-        match self {
-            CompileTimeHook::UserGoalExpansion | CompileTimeHook::GoalExpansion => {
-                clause_name!("goal_expansion")
-            }
-            CompileTimeHook::UserTermExpansion | CompileTimeHook::TermExpansion => {
-                clause_name!("term_expansion")
-            }
-        }
-    }
-
-    #[inline]
-    pub fn arity(self) -> usize {
-        match self {
-            CompileTimeHook::UserGoalExpansion | CompileTimeHook::GoalExpansion => 2,
-            CompileTimeHook::UserTermExpansion | CompileTimeHook::TermExpansion => 2,
-        }
-    }
-
-    #[inline]
-    pub fn user_scope(self) -> Self {
-        match self {
-            CompileTimeHook::UserGoalExpansion | CompileTimeHook::GoalExpansion => {
-                CompileTimeHook::UserGoalExpansion
-            }
-            CompileTimeHook::UserTermExpansion | CompileTimeHook::TermExpansion => {
-                CompileTimeHook::UserTermExpansion
-            }
-        }
-    }
-
-    #[inline]
-    pub fn has_module_scope(self) -> bool {
-        match self {
-            CompileTimeHook::UserTermExpansion | CompileTimeHook::UserGoalExpansion => false,
-            _ => true,
-        }
-    }
-}
 
 pub enum RefOrOwned<'a, T: 'a> {
     Borrowed(&'a T),
@@ -1094,7 +994,8 @@ impl<'a, T: 'a + fmt::Debug> fmt::Debug for RefOrOwned<'a, T> {
         match self {
             &RefOrOwned::Borrowed(ref borrowed) =>
                 write!(f, "Borrowed({:?})", borrowed),
-            &RefOrOwned::Owned(ref owned) => write!(f, "Owned({:?})", owned),
+            &RefOrOwned::Owned(ref owned) =>
+                write!(f, "Owned({:?})", owned),
         }
     }
 }
@@ -1107,9 +1008,7 @@ impl<'a, T> RefOrOwned<'a, T> {
         }
     }
 
-    pub fn to_owned(self) -> T
-    where
-        T: Clone,
+    pub fn to_owned(self) -> T where T: Clone
     {
         match self {
             RefOrOwned::Borrowed(item) => item.clone(),

src/machine/machine_state.rs

@@ -9,14 +9,14 @@ use crate::machine::copier::*;
 use crate::machine::heap::*;
 use crate::machine::machine_errors::*;
 use crate::machine::machine_indices::*;
-use crate::machine::modules::*;
+use crate::machine::partial_string::HeapPStrIter;
 use crate::machine::stack::*;
 use crate::machine::streams::*;
 use crate::rug::Integer;
 
 use crate::downcast::Any;
 
-use crate::indexmap::{IndexMap, IndexSet};
+use crate::indexmap::IndexMap;
 
 use std::cmp::Ordering;
 use std::convert::TryFrom;
@@ -25,292 +25,6 @@ use std::io::Write;
 use std::mem;
 use std::ops::{Index, IndexMut};
 
-#[derive(Debug)]
-pub(crate) struct HeapPStrIter<'a> {
-    focus: Addr,
-    machine_st: &'a MachineState,
-    seen: IndexSet<Addr>,
-}
-
-impl<'a> HeapPStrIter<'a> {
-    #[inline]
-    fn new(machine_st: &'a MachineState, focus: Addr) -> Self {
-        HeapPStrIter {
-            focus,
-            machine_st,
-            seen: IndexSet::new(),
-        }
-    }
-
-    #[inline]
-    pub(crate)
-    fn focus(&self) -> Addr {
-        self.machine_st.store(self.machine_st.deref(self.focus))
-    }
-
-    #[inline]
-    pub(crate)
-    fn to_string(&mut self) -> String {
-        let mut buf = String::new();
-
-        while let Some(iteratee) = self.next() {
-            match iteratee {
-                PStrIteratee::Char(c) => {
-                    buf.push(c);
-                }
-                PStrIteratee::PStrSegment(h, n) => {
-                    match &self.machine_st.heap[h] {
-                        HeapCellValue::PartialString(ref pstr, _) => {
-                            buf += pstr.as_str_from(n);
-                        }
-                        _ => {
-                            unreachable!()
-                        }
-                    }
-                }
-            }
-        }
-
-        buf
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-pub(crate) enum PStrIteratee {
-    Char(char),
-    PStrSegment(usize, usize),
-}
-
-impl<'a> Iterator for HeapPStrIter<'a> {
-    type Item = PStrIteratee;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let addr = self.machine_st.store(self.machine_st.deref(self.focus));
-
-        if !self.seen.contains(&addr) {
-            self.seen.insert(addr);
-        } else {
-            return None;
-        }
-
-        match addr {
-            Addr::PStrLocation(h, n) => {
-                if let &HeapCellValue::PartialString(_, has_tail) = &self.machine_st.heap[h] {
-                    self.focus = if has_tail {
-                        Addr::HeapCell(h + 1)
-                    } else {
-                        Addr::EmptyList
-                    };
-
-                    return Some(PStrIteratee::PStrSegment(h, n));
-                } else {
-                    unreachable!()
-                }
-            }
-            Addr::Lis(l) => {
-                let addr = self.machine_st.store(self.machine_st.deref(Addr::HeapCell(l)));
-
-                let opt_c = match addr {
-                    Addr::Con(h) if self.machine_st.heap.atom_at(h) => {
-                        if let HeapCellValue::Atom(ref atom, _) = &self.machine_st.heap[h] {
-                            if atom.is_char() {
-                                Some(atom.as_str().chars().next().unwrap())
-                            } else {
-                                None
-                            }
-                        } else {
-                            unreachable!()
-                        }
-                    }
-                    Addr::Char(c) => {
-                        Some(c)
-                    }
-                    _ => {
-                        None
-                    }
-                };
-
-                if let Some(c) = opt_c {
-                    self.focus = Addr::HeapCell(l + 1);
-                    return Some(PStrIteratee::Char(c));
-                } else {
-                    return None;
-                }
-            }
-            Addr::EmptyList => {
-                self.focus = Addr::EmptyList;
-                return None;
-            }
-            _ => {
-                return None;
-            }
-        }
-    }
-}
-
-#[inline]
-pub(super)
-fn compare_pstr_prefixes<'a>(
-    i1: &mut HeapPStrIter<'a>,
-    i2: &mut HeapPStrIter<'a>,
-) -> Option<Ordering> {
-    let mut r1 = i1.next();
-    let mut r2 = i2.next();
-
-    loop {
-        if let Some(r1i) = r1 {
-            if let Some(r2i) = r2 {
-                match (r1i, r2i) {
-                    (PStrIteratee::Char(c1), PStrIteratee::Char(c2)) => {
-                        if c1 != c2 {
-                            return c1.partial_cmp(&c2);
-                        }
-                    }
-                    (PStrIteratee::Char(c1), PStrIteratee::PStrSegment(h, n)) => {
-                        if let &HeapCellValue::PartialString(ref pstr, _) = &i2.machine_st.heap[h] {
-                            if let Some(c2) = pstr.as_str_from(n).chars().next() {
-                                if c1 != c2 {
-                                    return c1.partial_cmp(&c2);
-                                } else {
-                                    r1 = i1.next();
-                                    r2 = Some(PStrIteratee::PStrSegment(h, n + c2.len_utf8()));
-
-                                    continue;
-                                }
-                            } else {
-                                r2 = i2.next();
-                                continue;
-                            }
-                        } else {
-                            unreachable!()
-                        }
-                    }
-                    (PStrIteratee::PStrSegment(h, n), PStrIteratee::Char(c2)) => {
-                        if let &HeapCellValue::PartialString(ref pstr, _) = &i1.machine_st.heap[h] {
-                            if let Some(c1) = pstr.as_str_from(n).chars().next() {
-                                if c1 != c2 {
-                                    return c2.partial_cmp(&c1);
-                                } else {
-                                    r1 = i1.next();
-                                    r2 = Some(PStrIteratee::PStrSegment(h, n + c1.len_utf8()));
-
-                                    continue;
-                                }
-                            } else {
-                                r1 = i1.next();
-                                continue;
-                            }
-                        } else {
-                            unreachable!()
-                        }
-                    }
-                    (PStrIteratee::PStrSegment(h1, n1), PStrIteratee::PStrSegment(h2, n2)) => {
-                        match (&i1.machine_st.heap[h1], &i2.machine_st.heap[h2]) {
-                            (
-                                &HeapCellValue::PartialString(ref pstr1, _),
-                                &HeapCellValue::PartialString(ref pstr2, _),
-                            ) => {
-                                let str1 = pstr1.as_str_from(n1);
-                                let str2 = pstr2.as_str_from(n2);
-
-                                if str1.starts_with(str2) {
-                                    r1 = Some(PStrIteratee::PStrSegment(h1, n1 + str2.len()));
-                                    r2 = i2.next();
-
-                                    continue;
-                                } else if str2.starts_with(str1) {
-                                    r1 = i1.next();
-                                    r2 = Some(PStrIteratee::PStrSegment(h2, n2 + str1.len()));
-
-                                    continue;
-                                } else {
-                                    return str1.partial_cmp(str2);
-                                }
-                            }
-                            _ => {
-                                unreachable!()
-                            }
-                        }
-                    }
-                }
-
-                r1 = i1.next();
-                r2 = i2.next();
-
-                continue;
-            }
-        }
-
-        return match (i1.focus(), i2.focus()) {
-            (Addr::EmptyList, Addr::EmptyList) => {
-                Some(Ordering::Equal)
-            }
-            (Addr::EmptyList, _) => {
-                Some(Ordering::Less)
-            }
-            (_, Addr::EmptyList) => {
-                Some(Ordering::Greater)
-            }
-            _ => {
-                None
-            }
-        };
-    }
-}
-
-#[inline]
-pub(super)
-fn compare_pstr_to_string<'a>(
-    heap_pstr_iter: &mut HeapPStrIter<'a>,
-    s: &String,
-) -> Option<usize> {
-    let mut s_offset = 0;
-
-    while let Some(iteratee) = heap_pstr_iter.next() {
-        match iteratee {
-            PStrIteratee::Char(c1) => {
-                if let Some(c2) = s[s_offset ..].chars().next() {
-                    if c1 != c2 {
-                        return None;
-                    } else {
-                        s_offset += c1.len_utf8();
-                    }
-                } else {
-                    return Some(s_offset);
-                }
-            }
-            PStrIteratee::PStrSegment(h, n) => {
-                match heap_pstr_iter.machine_st.heap[h] {
-                    HeapCellValue::PartialString(ref pstr, _) => {
-                        let t = pstr.as_str_from(n);
-
-                        if s[s_offset ..].starts_with(t) {
-                            s_offset += t.len();
-                        } else if t.starts_with(&s[s_offset ..]) {
-                            heap_pstr_iter.focus =
-                                Addr::PStrLocation(h, n + s[s_offset ..].len());
-
-                            s_offset += s[s_offset ..].len();
-                            return Some(s_offset);
-                        } else {
-                            return None;
-                        }
-                    }
-                    _ => {
-                        unreachable!()
-                    }
-                }
-            }
-        }
-
-        if s[s_offset ..].is_empty() {
-            return Some(s_offset);
-        }
-    }
-
-    Some(s_offset)
-}
-
 #[derive(Debug)]
 pub struct Ball {
     pub(super) boundary: usize,
@@ -332,17 +46,6 @@ impl Ball {
         self.stub.clear();
     }
 
-    pub(super)
-    fn take(&mut self) -> Ball {
-        let boundary = self.boundary;
-        self.boundary = 0;
-
-        Ball {
-            boundary,
-            stub: self.stub.take(),
-        }
-    }
-
     pub(super)
     fn copy_and_align(&self, h: usize) -> Heap {
         let diff = self.boundary as i64 - h as i64;
@@ -586,6 +289,7 @@ impl Default for HeapPtr {
 
 #[derive(Debug)]
 pub struct MachineState {
+    pub(crate) atom_tbl: TabledData<Atom>,
     pub(super) s: HeapPtr,
     pub(super) p: CodePtr,
     pub(super) b: usize,
@@ -640,7 +344,7 @@ impl MachineState {
         loop {
             match self.read(
                 stream.clone(),
-                indices.atom_tbl.clone(),
+                self.atom_tbl.clone(),
                 &indices.op_dir,
             ) {
                 Ok(term_write_result) => {
@@ -654,7 +358,7 @@ impl MachineState {
                     let mut list_of_var_eqs = vec![];
 
                     for (var, binding) in term_write_result.var_dict.into_iter() {
-                        let var_atom = clause_name!(var.to_string(), indices.atom_tbl);
+                        let var_atom = clause_name!(var.to_string(), self.atom_tbl);
 
                         let h = self.heap.h();
                         let spec = fetch_atom_op_spec(clause_name!("="), None, &indices.op_dir);
@@ -854,6 +558,15 @@ impl MachineState {
         Ok(Some(printer))
     }
 
+    pub(super)
+    fn throw_undefined_error(&mut self, name: ClauseName, arity: usize) -> MachineStub {
+        let stub = MachineError::functor_stub(name.clone(), arity);
+        let h = self.heap.h();
+        let key = ExistenceError::Procedure(name, arity);
+
+        self.error_form(MachineError::existence_error(h, key), stub)
+    }
+
     #[inline]
     pub(crate)
     fn heap_pstr_iter<'a>(&'a self, focus: Addr) -> HeapPStrIter<'a> {
@@ -895,6 +608,24 @@ impl MachineState {
         Ok(chars)
     }
 
+    pub(super)
+    fn read_predicate_key(&self, name: Addr, arity: Addr) -> (ClauseName, usize) {
+        let predicate_name = atom_from!(self, self.store(self.deref(name)));
+        let arity = self.store(self.deref(arity));
+
+        let arity =
+            match Number::try_from((arity, &self.heap)) {
+                Ok(Number::Integer(n)) if &*n >= &0 && &*n <= &MAX_ARITY =>
+                    n.to_usize().unwrap(),
+                Ok(Number::Fixnum(n)) if n >= 0 && n <= MAX_ARITY as isize =>
+                    usize::try_from(n).unwrap(),
+                _ =>
+                    unreachable!()
+            };
+
+        (predicate_name, arity)
+    }
+
     pub(super)
     fn call_at_index(&mut self, arity: usize, p: LocalCodePtr) {
         self.cp.assign_if_local(self.p.clone() + 1);
@@ -914,58 +645,35 @@ impl MachineState {
     fn module_lookup(
         &mut self,
         indices: &IndexStore,
+        call_policy: &mut Box<dyn CallPolicy>,
         key: PredicateKey,
         module_name: ClauseName,
-        last_call: bool,
+        _last_call: bool,
+        current_input_stream: &mut Stream,
+        current_output_stream: &mut Stream,
     ) -> CallResult {
-        let (name, arity) = key;
-
-        if let Some(ref idx) = indices.get_code_index((name.clone(), arity), module_name.clone()) {
-            match idx.0.borrow().0 {
-                IndexPtr::Index(compiled_tl_index) => {
-                    if last_call {
-                        self.execute_at_index(arity, dir_entry!(compiled_tl_index));
-                    } else {
-                        self.call_at_index(arity, dir_entry!(compiled_tl_index));
-                    }
-
-                    return Ok(());
-                }
-                IndexPtr::DynamicUndefined => {
-                    self.fail = true;
-                    return Ok(());
-                }
-                IndexPtr::UserTermExpansion => {
-                    if last_call {
-                        self.execute_at_index(arity, LocalCodePtr::UserTermExpansion(0));
-                    } else {
-                        self.call_at_index(arity, LocalCodePtr::UserTermExpansion(0));
-                    }
-
-                    return Ok(());
-                }
-                IndexPtr::UserGoalExpansion => {
-                    if last_call {
-                        self.execute_at_index(arity, LocalCodePtr::UserGoalExpansion(0));
-                    } else {
-                        self.call_at_index(arity, LocalCodePtr::UserGoalExpansion(0));
-                    }
-
-                    return Ok(());
-                }
-                IndexPtr::InSituDirEntry(p) => {
-                    if last_call {
-                        self.execute_at_index(arity, LocalCodePtr::InSituDirEntry(p));
-                    } else {
-                        self.call_at_index(arity, LocalCodePtr::InSituDirEntry(p));
-                    }
-
-                    return Ok(());
-                }
-                _ => {}
-            }
+        if module_name.as_str() == "user" {
+            return call_policy.call_clause_type(
+                self,
+                key,
+                &indices.code_dir,
+                &indices.op_dir,
+                current_input_stream,
+                current_output_stream,
+            );
+        } else if let Some(module) = indices.modules.get(&module_name) {
+            return call_policy.call_clause_type(
+                self,
+                key,
+                &module.code_dir,
+                &module.op_dir,
+                current_input_stream,
+                current_output_stream,
+            );
         }
 
+        let (name, arity) = key;
+
         let h = self.heap.h();
         let stub = MachineError::functor_stub(name.clone(), arity);
         let err = MachineError::module_resolution_error(h, module_name, name, arity);
@@ -974,49 +682,6 @@ impl MachineState {
     }
 }
 
-fn try_in_situ_lookup(name: ClauseName, arity: usize, indices: &IndexStore) -> Option<LocalCodePtr>
-{
-    match indices.in_situ_code_dir.get(&(name.clone(), arity)) {
-        Some(p) => Some(LocalCodePtr::InSituDirEntry(*p)),
-        None =>
-            match indices.code_dir.get(&(name, arity)) {
-                Some(ref idx) => {
-                    if let IndexPtr::Index(p) = idx.0.borrow().0 {
-                        Some(LocalCodePtr::DirEntry(p))
-                    } else {
-                        None
-                    }
-                }
-                _ => None,
-            },
-    }
-}
-
-fn try_in_situ(
-    machine_st: &mut MachineState,
-    name: ClauseName,
-    arity: usize,
-    indices: &IndexStore,
-    last_call: bool,
-) -> CallResult {
-    if let Some(p) = try_in_situ_lookup(name.clone(), arity, indices) {
-        if last_call {
-            machine_st.execute_at_index(arity, p);
-        } else {
-            machine_st.call_at_index(arity, p);
-        }
-
-        machine_st.p = CodePtr::Local(p);
-        Ok(())
-    } else {
-        let stub = MachineError::functor_stub(name.clone(), arity);
-        let h = machine_st.heap.h();
-        let key = ExistenceError::Procedure(name, arity);
-
-        Err(machine_st.error_form(MachineError::existence_error(h, key), stub))
-    }
-}
-
 pub(crate) type CallResult = Result<(), Vec<HeapCellValue>>;
 
 pub(crate) trait CallPolicy: Any + fmt::Debug {
@@ -1090,7 +755,7 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         machine_st.attr_var_init.backtrack(attr_var_init_queue_b, attr_var_init_bindings_b);
 
         machine_st.hb = machine_st.heap.h();
-        machine_st.p += offset;
+        machine_st.p = CodePtr::Local(dir_entry!(machine_st.p.local().abs_loc() + offset));
 
         Ok(())
     }
@@ -1130,7 +795,7 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         machine_st.stack.truncate(b);
 
         machine_st.hb = machine_st.heap.h();
-        machine_st.p += offset;
+        machine_st.p = CodePtr::Local(dir_entry!(machine_st.p.local().abs_loc() + offset));
 
         Ok(())
     }
@@ -1180,13 +845,12 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         machine_st: &mut MachineState,
         name: ClauseName,
         arity: usize,
-        idx: CodeIndex,
-        indices: &mut IndexStore,
+        idx: &CodeIndex,
     ) -> CallResult {
         if machine_st.last_call {
-            self.try_execute(machine_st, name, arity, idx, indices)
+            self.try_execute(machine_st, name, arity, idx)
         } else {
-            self.try_call(machine_st, name, arity, idx, indices)
+            self.try_call(machine_st, name, arity, idx)
         }
     }
 
@@ -1195,27 +859,18 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         machine_st: &mut MachineState,
         name: ClauseName,
         arity: usize,
-        idx: CodeIndex,
-        indices: &IndexStore,
+        idx: &CodeIndex,
     ) -> CallResult {
-        match idx.0.borrow().0 {
+        match idx.get() {
             IndexPtr::DynamicUndefined => {
                 machine_st.fail = true;
+                return Ok(());
             }
             IndexPtr::Undefined => {
-                return try_in_situ(machine_st, name, arity, indices, false);
+                return Err(machine_st.throw_undefined_error(name, arity));
             }
             IndexPtr::Index(compiled_tl_index) => {
-                machine_st.call_at_index(arity, LocalCodePtr::DirEntry(compiled_tl_index))
-            }
-            IndexPtr::UserTermExpansion => {
-                machine_st.call_at_index(arity, LocalCodePtr::UserTermExpansion(0));
-            }
-            IndexPtr::UserGoalExpansion => {
-                machine_st.call_at_index(arity, LocalCodePtr::UserGoalExpansion(0));
-            }
-            IndexPtr::InSituDirEntry(p) => {
-                machine_st.call_at_index(arity, LocalCodePtr::InSituDirEntry(p));
+                machine_st.call_at_index(arity, LocalCodePtr::DirEntry(compiled_tl_index));
             }
         }
 
@@ -1227,26 +882,19 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         machine_st: &mut MachineState,
         name: ClauseName,
         arity: usize,
-        idx: CodeIndex,
-        indices: &IndexStore,
+        idx: &CodeIndex,
     ) -> CallResult {
-        match idx.0.borrow().0 {
-            IndexPtr::DynamicUndefined =>
-                machine_st.fail = true,
-            IndexPtr::Undefined =>
-                return try_in_situ(machine_st, name, arity, indices, true),
+        match idx.get() {
+            IndexPtr::DynamicUndefined => {
+                machine_st.fail = true;
+                return Ok(());
+            }
+            IndexPtr::Undefined => {
+                return Err(machine_st.throw_undefined_error(name, arity));
+            }
             IndexPtr::Index(compiled_tl_index) => {
                 machine_st.execute_at_index(arity, dir_entry!(compiled_tl_index))
             }
-            IndexPtr::UserTermExpansion => {
-                machine_st.execute_at_index(arity, LocalCodePtr::UserTermExpansion(0));
-            }
-            IndexPtr::UserGoalExpansion => {
-                machine_st.execute_at_index(arity, LocalCodePtr::UserGoalExpansion(0));
-            }
-            IndexPtr::InSituDirEntry(p) => {
-                machine_st.execute_at_index(arity, LocalCodePtr::InSituDirEntry(p));
-            }
         }
 
         Ok(())
@@ -1256,7 +904,8 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         &mut self,
         machine_st: &mut MachineState,
         ct: &BuiltInClauseType,
-        indices: &mut IndexStore,
+        _code_dir: &CodeDir,
+        op_dir: &OpDir,
         current_input_stream: &mut Stream,
         current_output_stream: &mut Stream,
     ) -> CallResult {
@@ -1305,15 +954,15 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
 
                 let atom = match machine_st.compare_term_test(&a2, &a3) {
                     Some(Ordering::Greater) => {
-                        let spec = fetch_atom_op_spec(clause_name!(">"), None, &indices.op_dir);
+                        let spec = fetch_atom_op_spec(clause_name!(">"), None, op_dir);
                         HeapCellValue::Atom(clause_name!(">"), spec)
                     }
                     Some(Ordering::Equal) => {
-                        let spec = fetch_atom_op_spec(clause_name!("="), None, &indices.op_dir);
+                        let spec = fetch_atom_op_spec(clause_name!("="), None, op_dir);
                         HeapCellValue::Atom(clause_name!("="), spec)
                     }
                     None | Some(Ordering::Less) => {
-                        let spec = fetch_atom_op_spec(clause_name!("<"), None, &indices.op_dir);
+                        let spec = fetch_atom_op_spec(clause_name!("<"), None, op_dir);
                         HeapCellValue::Atom(clause_name!("<"), spec)
                     }
                 };
@@ -1338,8 +987,8 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
             &BuiltInClauseType::Read => {
                 match machine_st.read(
                     current_input_stream.clone(),
-                    indices.atom_tbl.clone(),
-                    &indices.op_dir,
+                    machine_st.atom_tbl.clone(),
+                    op_dir,
                 ) {
                     Ok(offset) => {
                         let addr = machine_st[temp_v!(1)];
@@ -1347,16 +996,18 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
                     }
                     Err(ParserError::UnexpectedEOF) => {
                         let addr = machine_st[temp_v!(1)];
-                        let eof = clause_name!("end_of_file".to_string(),
-                            indices.atom_tbl);
+                        let eof = clause_name!("end_of_file".to_string(), machine_st.atom_tbl);
+
                         let atom = machine_st.heap.to_unifiable(
                             HeapCellValue::Atom(eof, None)
                         );
+
                         machine_st.unify(addr, atom);
                     }
                     Err(e) => {
                         let h = machine_st.heap.h();
                         let stub = MachineError::functor_stub(clause_name!("read"), 1);
+
                         let err = MachineError::syntax_error(h, e);
                         let err = machine_st.error_form(err, stub);
 
@@ -1382,7 +1033,7 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
                 return_from_clause!(machine_st.last_call, machine_st)
             }
             &BuiltInClauseType::Functor => {
-                machine_st.try_functor(&indices)?;
+                machine_st.try_functor(op_dir)?;
                 return_from_clause!(machine_st.last_call, machine_st)
             }
             &BuiltInClauseType::NotEq => {
@@ -1453,24 +1104,66 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         }
     }
 
-    fn compile_hook(
+    fn call_clause_type(
         &mut self,
         machine_st: &mut MachineState,
-        hook: &CompileTimeHook,
+        key: PredicateKey,
+        code_dir: &CodeDir,
+        op_dir: &OpDir,
+        current_input_stream: &mut Stream,
+        current_output_stream: &mut Stream,
     ) -> CallResult {
-        machine_st.cp = LocalCodePtr::TopLevel(0, 0);
+        let (name, arity) = key;
 
-        machine_st.num_of_args = hook.arity();
-        machine_st.b0 = machine_st.b;
+        match ClauseType::from(name.clone(), arity, None) {
+            ClauseType::BuiltIn(built_in) => {
+                machine_st.setup_built_in_call(built_in.clone());
+                self.call_builtin(
+                    machine_st,
+                    &built_in,
+                    code_dir,
+                    op_dir,
+                    current_input_stream,
+                    current_output_stream,
+                )?;
+            }
+            ClauseType::CallN => {
+                machine_st.handle_internal_call_n(arity);
 
-        machine_st.p = match hook {
-            CompileTimeHook::UserTermExpansion | CompileTimeHook::TermExpansion => {
-                CodePtr::Local(LocalCodePtr::UserTermExpansion(0))
+                if machine_st.fail {
+                    return Ok(());
+                }
+
+                machine_st.p = CodePtr::CallN(arity, machine_st.p.local(), machine_st.last_call);
             }
-            CompileTimeHook::UserGoalExpansion | CompileTimeHook::GoalExpansion => {
-                CodePtr::Local(LocalCodePtr::UserGoalExpansion(0))
+            ClauseType::Inlined(inlined) => {
+                machine_st.execute_inlined(&inlined);
+
+                if machine_st.last_call {
+                    machine_st.p = CodePtr::Local(machine_st.cp);
+                }
             }
-        };
+            ClauseType::Op(..) | ClauseType::Named(..) => {
+                if let Some(idx) = code_dir.get(&(name.clone(), arity)) {
+                    self.context_call(machine_st, name, arity, idx)?;
+                } else {
+                    return Err(machine_st.throw_undefined_error(name, arity));
+                }
+            }
+            ClauseType::System(_) => {
+                let name = functor!(clause_name(name));
+                let stub = MachineError::functor_stub(clause_name!("call"), arity + 1);
+
+                return Err(machine_st.error_form(
+                    MachineError::type_error(
+                        machine_st.heap.h(),
+                        ValidType::Callable,
+                        name
+                    ),
+                    stub,
+                ));
+            }
+        }
 
         Ok(())
     }
@@ -1479,57 +1172,20 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
         &mut self,
         machine_st: &mut MachineState,
         arity: usize,
-        indices: &mut IndexStore,
+        code_dir: &CodeDir,
+        op_dir: &OpDir,
         current_input_stream: &mut Stream,
         current_output_stream: &mut Stream,
     ) -> CallResult {
-        if let Some((name, arity)) = machine_st.setup_call_n(arity) {
-            match ClauseType::from(name.clone(), arity, None) {
-                ClauseType::BuiltIn(built_in) => {
-                    machine_st.setup_built_in_call(built_in.clone());
-                    self.call_builtin(
-                        machine_st,
-                        &built_in,
-                        indices,
-                        current_input_stream,
-                        current_output_stream,
-                    )?;
-                }
-                ClauseType::CallN => {
-                    machine_st.handle_internal_call_n(arity);
-
-                    if machine_st.fail {
-                        return Ok(());
-                    }
-
-                    machine_st.p = CodePtr::CallN(arity, machine_st.p.local(), machine_st.last_call);
-                }
-                ClauseType::Inlined(inlined) => {
-                    machine_st.execute_inlined(&inlined);
-
-                    if machine_st.last_call {
-                        machine_st.p = CodePtr::Local(machine_st.cp);
-                    }
-                }
-                ClauseType::Op(..) | ClauseType::Named(..) => {
-                    let module = name.owning_module();
-
-                    if let Some(idx) = indices.get_code_index((name.clone(), arity), module) {
-                        self.context_call(machine_st, name, arity, idx, indices)?;
-                    } else {
-                        try_in_situ(machine_st, name, arity, indices, machine_st.last_call)?;
-                    }
-                }
-                ClauseType::Hook(_) | ClauseType::System(_) => {
-                    let name = functor!(clause_name(name));
-                    let stub = MachineError::functor_stub(clause_name!("call"), arity + 1);
-
-                    return Err(machine_st.error_form(
-                        MachineError::type_error(machine_st.heap.h(), ValidType::Callable, name),
-                        stub,
-                    ));
-                }
-            };
+        if let Some(key) = machine_st.setup_call_n(arity) {
+            self.call_clause_type(
+                machine_st,
+                key,
+                code_dir,
+                op_dir,
+                current_input_stream,
+                current_output_stream,
+            )?;
         }
 
         Ok(())
@@ -1542,11 +1198,9 @@ impl CallPolicy for CWILCallPolicy {
         machine_st: &mut MachineState,
         name: ClauseName,
         arity: usize,
-        idx: CodeIndex,
-        indices: &mut IndexStore,
+        idx: &CodeIndex,
     ) -> CallResult {
-        self.prev_policy
-            .context_call(machine_st, name, arity, idx, indices)?;
+        self.prev_policy.context_call(machine_st, name, arity, idx)?;//, indices)?;
         self.increment(machine_st)
     }
 
@@ -1574,14 +1228,16 @@ impl CallPolicy for CWILCallPolicy {
         &mut self,
         machine_st: &mut MachineState,
         ct: &BuiltInClauseType,
-        indices: &mut IndexStore,
+        code_dir: &CodeDir,
+        op_dir: &OpDir,
         current_input_stream: &mut Stream,
         current_output_stream: &mut Stream,
     ) -> CallResult {
         self.prev_policy.call_builtin(
             machine_st,
             ct,
-            indices,
+            code_dir,
+            op_dir,
             current_input_stream,
             current_output_stream
         )?;
@@ -1593,14 +1249,16 @@ impl CallPolicy for CWILCallPolicy {
         &mut self,
         machine_st: &mut MachineState,
         arity: usize,
-        indices: &mut IndexStore,
+        code_dir: &CodeDir,
+        op_dir: &OpDir,
         current_input_stream: &mut Stream,
         current_output_stream: &mut Stream,
     ) -> CallResult {
         self.prev_policy.call_n(
             machine_st,
             arity,
-            indices,
+            code_dir,
+            op_dir,
             current_input_stream,
             current_output_stream,
         )?;

src/machine/machine_state_impl.rs

@@ -4,6 +4,7 @@ use crate::prolog_parser::tabled_rc::*;
 use crate::clause_types::*;
 use crate::forms::*;
 use crate::heap_iter::*;
+use crate::indexing::*;
 use crate::instructions::*;
 use crate::machine::INTERRUPT;
 use crate::machine::attributed_variables::*;
@@ -13,6 +14,7 @@ use crate::machine::heap::*;
 use crate::machine::machine_errors::*;
 use crate::machine::machine_indices::*;
 use crate::machine::machine_state::*;
+use crate::machine::partial_string::*;
 use crate::machine::stack::*;
 use crate::machine::streams::*;
 use crate::ordered_float::*;
@@ -24,22 +26,11 @@ use std::cmp::Ordering;
 use std::convert::TryFrom;
 use std::rc::Rc;
 
-macro_rules! try_or_fail {
-    ($s:ident, $e:expr) => {{
-        match $e {
-            Ok(val) => val,
-            Err(msg) => {
-                $s.throw_exception(msg);
-                return;
-            }
-        }
-    }};
-}
-
 impl MachineState {
     pub(crate)
     fn new() -> Self {
         MachineState {
+            atom_tbl: TabledData::new(Rc::new("".to_owned())),
             s: HeapPtr::default(),
             p: CodePtr::default(),
             b: 0,
@@ -67,36 +58,6 @@ impl MachineState {
         }
     }
 
-    pub(crate)
-    fn with_small_heap() -> Self {
-        MachineState {
-            s: HeapPtr::default(),
-            p: CodePtr::default(),
-            b: 0,
-            b0: 0,
-            e: 0,
-            num_of_args: 0,
-            cp: LocalCodePtr::default(),
-            attr_var_init: AttrVarInitializer::new(0, 0),
-            fail: false,
-            heap: Heap::new(),
-            mode: MachineMode::Write,
-            stack: Stack::new(),
-            registers: vec![Addr::HeapCell(0); MAX_ARITY + 1], // self.registers[0] is never used.
-            trail: vec![],
-            tr: 0,
-            hb: 0,
-            block: 0,
-            ball: Ball::new(),
-            lifted_heap: Heap::new(),
-            interms: vec![Number::default(); 0],
-            last_call: false,
-            heap_locs: HeapVarDict::new(),
-            flags: MachineFlags::default(),
-            at_end_of_expansion: false
-        }
-    }
-
     #[inline]
     pub fn machine_flags(&self) -> MachineFlags {
         self.flags
@@ -1390,85 +1351,130 @@ impl MachineState {
     }
 
     pub(super)
-    fn execute_indexing_instr(&mut self, instr: &IndexingInstruction) {
-        match instr {
-            &IndexingInstruction::SwitchOnTerm(arg, v, c, l, s) => {
-                let addr = self[temp_v!(arg)];
-                let addr = self.store(self.deref(addr));
+    fn execute_indexing_instr(
+        &mut self,
+        indexing_lines: &Vec<IndexingLine>,
+        call_policy: &mut Box<dyn CallPolicy>,
+    ) {
+        let mut index = 0;
+        let addr =
+            match &indexing_lines[0] {
+                &IndexingLine::Indexing(IndexingInstruction::SwitchOnTerm(arg, ..)) => {
+                    self.store(self.deref(self[temp_v!(arg)]))
+                }
+                _ => {
+                    unreachable!()
+                }
+            };
 
-                let offset = match addr {
-                    Addr::Stream(_) | Addr::TcpListener(_) => {
-                        0
-                    }
-                    Addr::HeapCell(_) | Addr::StackCell(..) | Addr::AttrVar(..) => {
-                        v
-                    }
-                    Addr::PStrLocation(..) => {
-                        l
-                    }
-                    Addr::Char(_) | Addr::Con(_) | Addr::CutPoint(_) |
-                    Addr::EmptyList | Addr::Fixnum(_) | Addr::Float(_) | Addr::Usize(_) => {
-                        c
-                    }
-                    Addr::Lis(_) => {
-                        l
-                    }
-                    Addr::Str(_) => {
-                        s
-                    }
-                };
-
-                match offset {
-                    0 => self.fail = true,
-                    o => self.p += o,
-                };
-            }
-            &IndexingInstruction::SwitchOnConstant(arg, _, ref hm) => {
-                let addr = self[temp_v!(arg)];
-                let addr = self.store(self.deref(addr));
-
-                let offset =
-                    match addr.as_constant_index(&self) {
-                        Some(c) => {
-                            match hm.get(&c) {
-                                Some(offset) => *offset,
-                                _ => 0,
-                            }
+        loop {
+            match &indexing_lines[index] {
+                &IndexingLine::Indexing(IndexingInstruction::SwitchOnTerm(_, v, c, l, s)) => {
+                    let offset = match addr {
+                        Addr::LoadStatePayload(_) | Addr::Stream(_) | Addr::TcpListener(_) => {
+                            IndexingCodePtr::Fail
                         }
-                        None => {
-                            0
+                        Addr::HeapCell(_) | Addr::StackCell(..) | Addr::AttrVar(..) => {
+                            IndexingCodePtr::External(v)
+                        }
+                        Addr::PStrLocation(..) => {
+                            l
+                        }
+                        Addr::Char(_) | Addr::Con(_) | Addr::CutPoint(_) |
+                        Addr::EmptyList | Addr::Fixnum(_) | Addr::Float(_) | Addr::Usize(_) => {
+                            c
+                        }
+                        Addr::Lis(_) => {
+                            l
+                        }
+                        Addr::Str(_) => {
+                            s
                         }
                     };
 
-                match offset {
-                    0 => self.fail = true,
-                    o => self.p += o,
-                };
-            }
-            &IndexingInstruction::SwitchOnStructure(arg, _, ref hm) => {
-                let a1 = self.registers[arg];
-                let addr = self.store(self.deref(a1));
-
-                let offset = match addr {
-                    Addr::Str(s) => {
-                        if let &HeapCellValue::NamedStr(arity, ref name, _) = &self.heap[s] {
-                            match hm.get(&(name.clone(), arity)) {
-                                Some(offset) => *offset,
-                                _ => 0,
+                    match offset {
+                        IndexingCodePtr::Fail => {
+                            self.fail = true;
+                            break;
+                        }
+                        IndexingCodePtr::External(o) => {
+                            self.p += o;
+                            break;
+                        }
+                        IndexingCodePtr::Internal(o) => {
+                            index += o;
+                        }
+                    };
+                }
+                &IndexingLine::Indexing(IndexingInstruction::SwitchOnConstant(ref hm)) => {
+                    let offset =
+                        match addr.as_constant_index(&self) {
+                            Some(c) => {
+                                match hm.get(&c) {
+                                    Some(offset) => *offset,
+                                    _ => IndexingCodePtr::Fail,
+                                }
                             }
-                        } else {
-                            0
+                            None => {
+                                IndexingCodePtr::Fail
+                            }
+                        };
+
+                    match offset {
+                        IndexingCodePtr::Fail => {
+                            self.fail = true;
+                            break;
+                        }
+                        IndexingCodePtr::External(o) => {
+                            self.p += o;
+                            break;
+                        }
+                        IndexingCodePtr::Internal(o) => {
+                            index += o;
+                        }
+                    };
+                }
+                &IndexingLine::Indexing(IndexingInstruction::SwitchOnStructure(ref hm)) => {
+                    let offset = match addr {
+                        Addr::Str(s) => {
+                            if let &HeapCellValue::NamedStr(arity, ref name, _) = &self.heap[s] {
+                                match hm.get(&(name.clone(), arity)) {
+                                    Some(offset) => *offset,
+                                    _ => IndexingCodePtr::Fail
+                                }
+                            } else {
+                                IndexingCodePtr::Fail
+                            }
+                        }
+                        _ => {
+                            IndexingCodePtr::Fail
+                        }
+                    };
+
+                    match offset {
+                        IndexingCodePtr::Fail => {
+                            self.fail = true;
+                            break;
+                        }
+                        IndexingCodePtr::External(o) => {
+                            self.p += o;
+                            break;
+                        }
+                        IndexingCodePtr::Internal(o) => {
+                            index += o;
                         }
                     }
-                    _ => {
-                        0
+                }
+                &IndexingLine::IndexedChoice(ref instrs) => {
+                    if let LocalCodePtr::DirEntry(p) = self.p.local() {
+                        self.p = CodePtr::Local(LocalCodePtr::IndexingBuf(p, index, 0));
+                    } else {
+                        unreachable!()
                     }
-                };
 
-                match offset {
-                    0 => self.fail = true,
-                    o => self.p += o,
-                };
+                    self.execute_indexed_choice_instr(instrs.first().unwrap(), call_policy);
+                    break;
+                }
             }
         };
     }
@@ -1679,23 +1685,38 @@ impl MachineState {
         Some((name, arity + narity - 1))
     }
 
-    pub(super) fn unwind_stack(&mut self) {
+    pub(super)
+    fn unwind_stack(&mut self) {
         self.b = self.block;
         self.fail = true;
     }
 
-    pub(crate) fn is_cyclic_term(&self, addr: Addr) -> bool {
+    pub(crate)
+    fn is_cyclic_term(&self, addr: Addr) -> bool {
         let mut seen = IndexSet::new();
         let mut fail = false;
         let mut iter = self.pre_order_iter(addr);
 
+        let is_composite = |addr: &Addr| {
+            match *addr {
+                Addr::Str(_) | Addr::Lis(_) | Addr::PStrLocation(..) => {
+                    true
+                }
+                _ => {
+                    false
+                }
+            }
+        };
+
         loop {
             if let Some(addr) = iter.stack().last() {
-                if !seen.contains(addr) {
-                    seen.insert(*addr);
-                } else {
-                    fail = true;
-                    break;
+                if is_composite(addr) {
+                    if !seen.contains(addr) {
+                        seen.insert(*addr);
+                    } else {
+                        fail = true;
+                        break;
+                    }
                 }
             }
 
@@ -2513,7 +2534,7 @@ impl MachineState {
     }
 
     pub(super)
-    fn try_functor(&mut self, indices: &IndexStore) -> CallResult {
+    fn try_functor(&mut self, op_dir: &OpDir) -> CallResult {
         let stub = MachineError::functor_stub(clause_name!("functor"), 3);
         let a1 = self.store(self.deref(self[temp_v!(1)]));
 
@@ -2531,7 +2552,7 @@ impl MachineState {
                         name.clone(),
                         arity,
                         spec,
-                        &indices.op_dir,
+                        &op_dir,
                     );
 
                     self.try_functor_compound_case(name, arity, spec)
@@ -2545,7 +2566,7 @@ impl MachineState {
                     clause_name!("."),
                     2,
                     None,
-                    &indices.op_dir,
+                    &op_dir,
                 );
 
                 self.try_functor_compound_case(clause_name!("."), 2, spec)
@@ -2620,7 +2641,7 @@ impl MachineState {
                                 name,
                                 arity as usize,
                                 spec,
-                                &indices.op_dir,
+                                &op_dir,
                                 a1.as_var().unwrap(),
                             );
                         } else {
@@ -2637,10 +2658,10 @@ impl MachineState {
                     }
                     Addr::Char(c) => {
                         self.try_functor_fabricate_struct(
-                            clause_name!(c.to_string(), indices.atom_tbl),
+                            clause_name!(c.to_string(), self.atom_tbl),
                             arity as usize,
                             None,
-                            &indices.op_dir,
+                            &op_dir,
                             a1.as_var().unwrap(),
                         );
                     }
@@ -3156,16 +3177,23 @@ impl MachineState {
                 call_policy.call_builtin(
                     self,
                     ct,
-                    indices,
+                    &indices.code_dir,
+                    &indices.op_dir,
                     current_input_stream,
                     current_output_stream,
                 )
             ),
             &ClauseType::CallN => try_or_fail!(
                 self,
-                call_policy.call_n(self, arity, indices, current_input_stream, current_output_stream)
+                call_policy.call_n(
+                    self,
+                    arity,
+                    &indices.code_dir,
+                    &indices.op_dir,
+                    current_input_stream,
+                    current_output_stream,
+                )
             ),
-            &ClauseType::Hook(ref hook) => try_or_fail!(self, call_policy.compile_hook(self, hook)),
             &ClauseType::Inlined(ref ct) => {
                 self.execute_inlined(ct);
 
@@ -3176,7 +3204,7 @@ impl MachineState {
             &ClauseType::Named(ref name, _, ref idx) | &ClauseType::Op(ref name, _, ref idx) => {
                 try_or_fail!(
                     self,
-                    call_policy.context_call(self, name.clone(), arity, idx.clone(), indices)
+                    call_policy.context_call(self, name.clone(), arity, idx)
                 )
             }
             &ClauseType::System(ref ct) => try_or_fail!(
@@ -3234,6 +3262,9 @@ impl MachineState {
                 self.b0 = self.b;
                 self.p += offset;
             }
+            &ControlInstruction::RevJmpBy(offset) => {
+                self.p -= offset;
+            }
             &ControlInstruction::Proceed => {
                 self.p = CodePtr::Local(self.cp);
             }
@@ -3272,7 +3303,7 @@ impl MachineState {
                 }
 
                 self.hb = self.heap.h();
-                self.p += offset;
+                self.p = CodePtr::Local(dir_entry!(self.p.local().abs_loc() + offset));
             }
             &IndexedChoiceInstruction::Retry(l) => {
                 try_or_fail!(self, call_policy.retry(self, l));
@@ -3283,7 +3314,8 @@ impl MachineState {
         };
     }
 
-    pub(super) fn execute_choice_instr(
+    pub(super)
+    fn execute_choice_instr(
         &mut self,
         instr: &ChoiceInstruction,
         call_policy: &mut Box<dyn CallPolicy>,
@@ -3320,14 +3352,14 @@ impl MachineState {
                 let mut call_policy = DefaultCallPolicy {};
                 try_or_fail!(self, call_policy.retry_me_else(self, offset))
             }
-            &ChoiceInstruction::DefaultTrustMe => {
+            &ChoiceInstruction::DefaultTrustMe(_) => {
                 let mut call_policy = DefaultCallPolicy {};
                 try_or_fail!(self, call_policy.trust_me(self))
             }
             &ChoiceInstruction::RetryMeElse(offset) => {
                 try_or_fail!(self, call_policy.retry_me_else(self, offset))
             }
-            &ChoiceInstruction::TrustMe => {
+            &ChoiceInstruction::TrustMe(_) => {
                 try_or_fail!(self, call_policy.trust_me(self))
             }
         }
@@ -3374,30 +3406,4 @@ impl MachineState {
             }
         }
     }
-
-    pub fn reset(&mut self) {
-        self.stack.drop_in_place();
-
-        self.hb = 0;
-        self.e = 0;
-        self.b = 0;
-        self.b0 = 0;
-        self.s = HeapPtr::default();
-        self.tr = 0;
-        self.p = CodePtr::default();
-        self.cp = LocalCodePtr::default();
-        self.attr_var_init.reset();
-        self.num_of_args = 0;
-
-        self.fail = false;
-        self.trail.clear();
-        self.heap.clear();
-        self.mode = MachineMode::Write;
-        self.registers = vec![Addr::HeapCell(0); MAX_ARITY + 1]; // self.registers[0] is never used.
-        self.block = 0;
-
-        self.ball.reset();
-        self.heap_locs.clear();
-        self.lifted_heap.clear();
-    }
 }

src/machine/modules.rs

@@ -1,371 +0,0 @@
-use crate::prolog_parser::ast::*;
-use crate::prolog_parser::tabled_rc::*;
-
-use crate::forms::*;
-use crate::machine::code_repo::*;
-use crate::machine::machine_errors::*;
-use crate::machine::machine_indices::*;
-
-use std::collections::VecDeque;
-use std::mem;
-
-// Module's and related types are defined in forms.
-impl Module {
-    pub fn new(
-        module_decl: ModuleDecl,
-        atom_tbl: TabledData<Atom>,
-        listing_src: ListingSource,
-    ) -> Self
-    {
-        Module {
-            atom_tbl,
-            module_decl,
-            term_dir: TermDir::new(),
-            user_term_expansions: (Predicate::new(), VecDeque::from(vec![])),
-            user_goal_expansions: (Predicate::new(), VecDeque::from(vec![])),
-            term_expansions: (Predicate::new(), VecDeque::from(vec![])),
-            goal_expansions: (Predicate::new(), VecDeque::from(vec![])),
-            local_term_expansions: (Predicate::new(), VecDeque::from(vec![])),
-            local_goal_expansions: (Predicate::new(), VecDeque::from(vec![])),
-            code_dir: CodeDir::new(),
-            op_dir: default_op_dir(),
-            inserted_expansions: false,
-            is_impromptu_module: false,
-            listing_src,
-        }
-    }
-
-    pub fn dump_expansions(
-        &self,
-        code_repo: &mut CodeRepo,
-    ) -> Result<(), ParserError> {
-        {
-            let te = code_repo
-                .term_dir
-                .entry((clause_name!("term_expansion"), 2))
-                .or_insert((Predicate::new(), VecDeque::from(vec![])));
-
-            (te.0)
-                .0
-                .extend((self.user_term_expansions.0).0.iter().cloned());
-
-            te.1.extend(self.user_term_expansions.1.iter().cloned());
-        }
-
-        {
-            let ge = code_repo
-                .term_dir
-                .entry((clause_name!("goal_expansion"), 2))
-                .or_insert((Predicate::new(), VecDeque::from(vec![])));
-
-            (ge.0)
-                .0
-                .extend((self.user_goal_expansions.0).0.iter().cloned());
-
-            ge.1.extend(self.user_goal_expansions.1.iter().cloned());
-        }
-
-        code_repo.compile_hook(CompileTimeHook::TermExpansion)?;
-        code_repo.compile_hook(CompileTimeHook::GoalExpansion)?;
-
-        Ok(())
-    }
-
-    pub fn add_expansion_record(
-        &mut self,
-        hook: CompileTimeHook,
-        clause: PredicateClause,
-        queue: VecDeque<TopLevel>,
-    ) {
-        match hook {
-            CompileTimeHook::TermExpansion | CompileTimeHook::UserTermExpansion => {
-                (self.term_expansions.0).0.push(clause);
-                self.term_expansions.1.extend(queue.into_iter());
-            }
-            CompileTimeHook::GoalExpansion | CompileTimeHook::UserGoalExpansion => {
-                (self.goal_expansions.0).0.push(clause);
-                self.goal_expansions.1.extend(queue.into_iter());
-            }
-        }
-    }
-
-    pub fn add_local_expansion(
-        &mut self,
-        hook: CompileTimeHook,
-        clause: PredicateClause,
-        queue: VecDeque<TopLevel>,
-    ) {
-        match hook {
-            CompileTimeHook::TermExpansion => {
-                (self.local_term_expansions.0).0.push(clause);
-                self.local_term_expansions.1.extend(queue.into_iter());
-            }
-            CompileTimeHook::GoalExpansion => {
-                (self.local_goal_expansions.0).0.push(clause);
-                self.local_goal_expansions.1.extend(queue.into_iter());
-            }
-            _ => {}
-        }
-    }
-
-    pub fn take_local_expansions(&mut self) -> Vec<(Predicate, VecDeque<TopLevel>)>
-    {
-        let term_expansions =
-            mem::replace(&mut self.local_term_expansions, (Predicate::new(), VecDeque::new()));
-        let goal_expansions =
-            mem::replace(&mut self.local_goal_expansions, (Predicate::new(), VecDeque::new()));
-
-        vec![term_expansions, goal_expansions]
-    }
-}
-
-pub trait SubModuleUser {
-    fn atom_tbl(&self) -> TabledData<Atom>;
-    fn op_dir(&mut self) -> &mut OpDir;
-    fn remove_code_index(&mut self, _: PredicateKey);
-    fn get_code_index(&self, _: PredicateKey, _: ClauseName) -> Option<CodeIndex>;
-
-    fn insert_dir_entry(&mut self, _: ClauseName, _: usize, _: CodeIndex);
-
-    fn get_op_module_name(&mut self, name: ClauseName, fixity: Fixity) -> Option<ClauseName> {
-        self.op_dir()
-            .get(&(name, fixity))
-            .map(|op_val| op_val.owning_module())
-    }
-
-    fn remove_module(&mut self, mod_name: ClauseName, module: &Module) {
-        for export in module.module_decl.exports.iter().cloned() {
-            match export {
-                ModuleExport::PredicateKey((name, arity)) => {
-                    let name = name.defrock_brackets();
-
-                    match self.get_code_index((name.clone(), arity), mod_name.clone()) {
-                        Some(CodeIndex(ref code_idx)) => {
-                            if &code_idx.borrow().1 != &module.module_decl.name {
-                                continue;
-                            }
-
-                            self.remove_code_index((name.clone(), arity));
-
-                            // remove or respecify ops.
-                            if arity == 2 {
-                                if let Some(mod_name) = self.get_op_module_name(name.clone(), Fixity::In) {
-                                    if mod_name == module.module_decl.name {
-                                        self.op_dir().remove(&(name.clone(), Fixity::In));
-                                    }
-                                }
-                            } else if arity == 1 {
-                                if let Some(mod_name) = self.get_op_module_name(name.clone(), Fixity::Pre) {
-                                    if mod_name == module.module_decl.name {
-                                        self.op_dir().remove(&(name.clone(), Fixity::Pre));
-                                    }
-                                }
-
-                                if let Some(mod_name) = self.get_op_module_name(name.clone(), Fixity::Post)
-                                {
-                                    if mod_name == module.module_decl.name {
-                                        self.op_dir().remove(&(name.clone(), Fixity::Post));
-                                    }
-                                }
-                            }
-                        }
-                        _ => {}
-                    };
-                },
-                ModuleExport::OpDecl(op_decl) => {
-                    let op_dir = self.op_dir();
-                    op_dir.remove(&(op_decl.name(), op_decl.fixity()));
-                }
-            }
-        }
-    }
-
-    // returns true on successful import.
-    fn import_decl(&mut self, name: ClauseName, arity: usize, submodule: &Module) -> bool {
-        let name = name.defrock_brackets();
-
-        if let Some(code_data) = submodule.code_dir.get(&(name.clone(), arity)) {
-            let name = name.with_table(submodule.atom_tbl.clone());
-            let atom_tbl = self.atom_tbl();
-
-            atom_tbl.borrow_mut().insert(name.to_rc());
-
-            self.insert_dir_entry(name, arity, code_data.clone());
-            true
-        } else {
-            submodule.is_impromptu_module
-        }
-    }
-
-    fn use_qualified_module(
-        &mut self,
-        _: &mut CodeRepo,
-        _: MachineFlags,
-        _: &Module,
-        _: &Vec<ModuleExport>,
-    ) -> Result<(), SessionError>;
-
-    fn use_module(
-        &mut self,
-        _: &mut CodeRepo,
-        _: MachineFlags,
-        _: &Module
-    ) -> Result<(), SessionError>;
-}
-
-pub fn use_qualified_module<User>(
-    user: &mut User,
-    submodule: &Module,
-    exports: &Vec<ModuleExport>,
-) -> Result<(), SessionError>
-where
-    User: SubModuleUser,
-{
-    for export in exports.iter().cloned() {
-        match export {
-            ModuleExport::PredicateKey((name, arity)) => {
-                if !submodule
-                    .module_decl
-                    .exports
-                    .contains(&ModuleExport::PredicateKey((name.clone(), arity)))
-                {
-                    continue;
-                }
-
-                if !user.import_decl(name.clone(), arity, submodule) {
-                    let submodule_name = submodule.module_decl.name.clone();
-
-                    return Err(SessionError::ModuleDoesNotContainExport(
-                        submodule_name,
-                        (name, arity)
-                    ));
-                }
-            },
-            ModuleExport::OpDecl(op_decl) => {
-                if !submodule
-                    .module_decl
-                    .exports
-                    .contains(&ModuleExport::OpDecl(op_decl.clone()))
-                {
-                    continue;
-                }
-
-                let op_dir = user.op_dir();
-                let prec = op_decl.0;
-
-                op_decl.insert_into_op_dir(
-                    submodule.module_decl.name.clone(),
-                    op_dir,
-                    prec,
-                );
-            }
-        }
-    }
-
-    Ok(())
-}
-
-pub fn use_module<User: SubModuleUser>(
-    user: &mut User,
-    submodule: &Module,
-) -> Result<(), SessionError> {
-    for export in submodule.module_decl.exports.iter().cloned() {
-        match export {
-            ModuleExport::PredicateKey((name, arity)) => {
-                if !user.import_decl(name.clone(), arity, submodule) {
-                    let submodule_name = submodule.module_decl.name.clone();
-
-                    return Err(SessionError::ModuleDoesNotContainExport(
-                        submodule_name,
-                        (name, arity)
-                    ));
-                }
-            }
-            ModuleExport::OpDecl(op_decl) => {
-                let op_dir = user.op_dir();
-                let prec = op_decl.0;
-
-                op_decl.insert_into_op_dir(
-                    submodule.module_decl.name.clone(),
-                    op_dir,
-                    prec,
-                );
-            }
-        }
-    }
-
-    Ok(())
-}
-
-impl SubModuleUser for Module {
-    fn atom_tbl(&self) -> TabledData<Atom> {
-        self.atom_tbl.clone()
-    }
-
-    fn op_dir(&mut self) -> &mut OpDir {
-        &mut self.op_dir
-    }
-
-    fn get_code_index(&self, key: PredicateKey, _: ClauseName) -> Option<CodeIndex> {
-        self.code_dir.get(&key).cloned()
-    }
-
-    fn remove_code_index(&mut self, key: PredicateKey) {
-        self.code_dir.remove(&key);
-    }
-
-    fn insert_dir_entry(&mut self, name: ClauseName, arity: usize, idx: CodeIndex) {
-        self.code_dir.insert((name, arity), idx);
-    }
-
-    fn use_qualified_module(
-        &mut self,
-        _: &mut CodeRepo,
-        _: MachineFlags,
-        submodule: &Module,
-        exports: &Vec<ModuleExport>,
-    ) -> Result<(), SessionError> {
-        use_qualified_module(self, submodule, exports)?;
-
-        (self.user_term_expansions.0)
-            .0
-            .extend((submodule.term_expansions.0).0.iter().cloned());
-        self.user_term_expansions
-            .1
-            .extend(submodule.term_expansions.1.iter().cloned());
-
-        (self.user_goal_expansions.0)
-            .0
-            .extend((submodule.goal_expansions.0).0.iter().cloned());
-        self.user_goal_expansions
-            .1
-            .extend(submodule.goal_expansions.1.iter().cloned());
-
-        Ok(())
-    }
-
-    fn use_module(
-        &mut self,
-        _: &mut CodeRepo,
-        _: MachineFlags,
-        submodule: &Module,
-    ) -> Result<(), SessionError> {
-        use_module(self, submodule)?;
-
-        (self.user_term_expansions.0)
-            .0
-            .extend((submodule.term_expansions.0).0.iter().cloned());
-        self.user_term_expansions
-            .1
-            .extend(submodule.term_expansions.1.iter().cloned());
-
-        (self.user_goal_expansions.0)
-            .0
-            .extend((submodule.goal_expansions.0).0.iter().cloned());
-        self.user_goal_expansions
-            .1
-            .extend(submodule.goal_expansions.1.iter().cloned());
-
-        Ok(())
-    }
-}

src/machine/partial_string.rs

@@ -1,12 +1,18 @@
+use crate::machine::*;
+use crate::machine::machine_indices::*;
+
 use core::marker::PhantomData;
 
 use std::alloc;
+use std::cmp::Ordering;
 use std::mem;
 use std::ptr;
 use std::ops::RangeFrom;
 use std::slice;
 use std::str;
 
+use indexmap::IndexSet;
+
 #[derive(Debug)]
 pub struct PartialString {
     buf: *const u8,
@@ -200,3 +206,290 @@ impl PartialString {
         }
     }
 }
+
+#[derive(Debug)]
+pub(crate) struct HeapPStrIter<'a> {
+    focus: Addr,
+    machine_st: &'a MachineState,
+    seen: IndexSet<Addr>,
+}
+
+impl<'a> HeapPStrIter<'a> {
+    #[inline]
+    pub(super)
+    fn new(machine_st: &'a MachineState, focus: Addr) -> Self {
+        HeapPStrIter {
+            focus,
+            machine_st,
+            seen: IndexSet::new(),
+        }
+    }
+
+    #[inline]
+    pub(crate)
+    fn focus(&self) -> Addr {
+        self.machine_st.store(self.machine_st.deref(self.focus))
+    }
+
+    #[inline]
+    pub(crate)
+    fn to_string(&mut self) -> String {
+        let mut buf = String::new();
+
+        while let Some(iteratee) = self.next() {
+            match iteratee {
+                PStrIteratee::Char(c) => {
+                    buf.push(c);
+                }
+                PStrIteratee::PStrSegment(h, n) => {
+                    match &self.machine_st.heap[h] {
+                        HeapCellValue::PartialString(ref pstr, _) => {
+                            buf += pstr.as_str_from(n);
+                        }
+                        _ => {
+                            unreachable!()
+                        }
+                    }
+                }
+            }
+        }
+
+        buf
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum PStrIteratee {
+    Char(char),
+    PStrSegment(usize, usize),
+}
+
+impl<'a> Iterator for HeapPStrIter<'a> {
+    type Item = PStrIteratee;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let addr = self.machine_st.store(self.machine_st.deref(self.focus));
+
+        if !self.seen.contains(&addr) {
+            self.seen.insert(addr);
+        } else {
+            return None;
+        }
+
+        match addr {
+            Addr::PStrLocation(h, n) => {
+                if let &HeapCellValue::PartialString(_, has_tail) = &self.machine_st.heap[h] {
+                    self.focus = if has_tail {
+                        Addr::HeapCell(h + 1)
+                    } else {
+                        Addr::EmptyList
+                    };
+
+                    return Some(PStrIteratee::PStrSegment(h, n));
+                } else {
+                    unreachable!()
+                }
+            }
+            Addr::Lis(l) => {
+                let addr = self.machine_st.store(self.machine_st.deref(Addr::HeapCell(l)));
+
+                let opt_c = match addr {
+                    Addr::Con(h) if self.machine_st.heap.atom_at(h) => {
+                        if let HeapCellValue::Atom(ref atom, _) = &self.machine_st.heap[h] {
+                            if atom.is_char() {
+                                Some(atom.as_str().chars().next().unwrap())
+                            } else {
+                                None
+                            }
+                        } else {
+                            unreachable!()
+                        }
+                    }
+                    Addr::Char(c) => {
+                        Some(c)
+                    }
+                    _ => {
+                        None
+                    }
+                };
+
+                if let Some(c) = opt_c {
+                    self.focus = Addr::HeapCell(l + 1);
+                    return Some(PStrIteratee::Char(c));
+                } else {
+                    return None;
+                }
+            }
+            Addr::EmptyList => {
+                self.focus = Addr::EmptyList;
+                return None;
+            }
+            _ => {
+                return None;
+            }
+        }
+    }
+}
+
+#[inline]
+pub(super)
+fn compare_pstr_prefixes<'a>(
+    i1: &mut HeapPStrIter<'a>,
+    i2: &mut HeapPStrIter<'a>,
+) -> Option<Ordering> {
+    let mut r1 = i1.next();
+    let mut r2 = i2.next();
+
+    loop {
+        if let Some(r1i) = r1 {
+            if let Some(r2i) = r2 {
+                match (r1i, r2i) {
+                    (PStrIteratee::Char(c1), PStrIteratee::Char(c2)) => {
+                        if c1 != c2 {
+                            return c1.partial_cmp(&c2);
+                        }
+                    }
+                    (PStrIteratee::Char(c1), PStrIteratee::PStrSegment(h, n)) => {
+                        if let &HeapCellValue::PartialString(ref pstr, _) = &i2.machine_st.heap[h] {
+                            if let Some(c2) = pstr.as_str_from(n).chars().next() {
+                                if c1 != c2 {
+                                    return c1.partial_cmp(&c2);
+                                } else {
+                                    r1 = i1.next();
+                                    r2 = Some(PStrIteratee::PStrSegment(h, n + c2.len_utf8()));
+
+                                    continue;
+                                }
+                            } else {
+                                r2 = i2.next();
+                                continue;
+                            }
+                        } else {
+                            unreachable!()
+                        }
+                    }
+                    (PStrIteratee::PStrSegment(h, n), PStrIteratee::Char(c2)) => {
+                        if let &HeapCellValue::PartialString(ref pstr, _) = &i1.machine_st.heap[h] {
+                            if let Some(c1) = pstr.as_str_from(n).chars().next() {
+                                if c1 != c2 {
+                                    return c2.partial_cmp(&c1);
+                                } else {
+                                    r1 = i1.next();
+                                    r2 = Some(PStrIteratee::PStrSegment(h, n + c1.len_utf8()));
+
+                                    continue;
+                                }
+                            } else {
+                                r1 = i1.next();
+                                continue;
+                            }
+                        } else {
+                            unreachable!()
+                        }
+                    }
+                    (PStrIteratee::PStrSegment(h1, n1), PStrIteratee::PStrSegment(h2, n2)) => {
+                        match (&i1.machine_st.heap[h1], &i2.machine_st.heap[h2]) {
+                            (
+                                &HeapCellValue::PartialString(ref pstr1, _),
+                                &HeapCellValue::PartialString(ref pstr2, _),
+                            ) => {
+                                let str1 = pstr1.as_str_from(n1);
+                                let str2 = pstr2.as_str_from(n2);
+
+                                if str1.starts_with(str2) {
+                                    r1 = Some(PStrIteratee::PStrSegment(h1, n1 + str2.len()));
+                                    r2 = i2.next();
+
+                                    continue;
+                                } else if str2.starts_with(str1) {
+                                    r1 = i1.next();
+                                    r2 = Some(PStrIteratee::PStrSegment(h2, n2 + str1.len()));
+
+                                    continue;
+                                } else {
+                                    return str1.partial_cmp(str2);
+                                }
+                            }
+                            _ => {
+                                unreachable!()
+                            }
+                        }
+                    }
+                }
+
+                r1 = i1.next();
+                r2 = i2.next();
+
+                continue;
+            }
+        }
+
+        return match (i1.focus(), i2.focus()) {
+            (Addr::EmptyList, Addr::EmptyList) => {
+                Some(Ordering::Equal)
+            }
+            (Addr::EmptyList, _) => {
+                Some(Ordering::Less)
+            }
+            (_, Addr::EmptyList) => {
+                Some(Ordering::Greater)
+            }
+            _ => {
+                None
+            }
+        };
+    }
+}
+
+#[inline]
+pub(super)
+fn compare_pstr_to_string<'a>(
+    heap_pstr_iter: &mut HeapPStrIter<'a>,
+    s: &String,
+) -> Option<usize> {
+    let mut s_offset = 0;
+
+    while let Some(iteratee) = heap_pstr_iter.next() {
+        match iteratee {
+            PStrIteratee::Char(c1) => {
+                if let Some(c2) = s[s_offset ..].chars().next() {
+                    if c1 != c2 {
+                        return None;
+                    } else {
+                        s_offset += c1.len_utf8();
+                    }
+                } else {
+                    return Some(s_offset);
+                }
+            }
+            PStrIteratee::PStrSegment(h, n) => {
+                match heap_pstr_iter.machine_st.heap[h] {
+                    HeapCellValue::PartialString(ref pstr, _) => {
+                        let t = pstr.as_str_from(n);
+
+                        if s[s_offset ..].starts_with(t) {
+                            s_offset += t.len();
+                        } else if t.starts_with(&s[s_offset ..]) {
+                            heap_pstr_iter.focus =
+                                Addr::PStrLocation(h, n + s[s_offset ..].len());
+
+                            s_offset += s[s_offset ..].len();
+                            return Some(s_offset);
+                        } else {
+                            return None;
+                        }
+                    }
+                    _ => {
+                        unreachable!()
+                    }
+                }
+            }
+        }
+
+        if s[s_offset ..].is_empty() {
+            return Some(s_offset);
+        }
+    }
+
+    Some(s_offset)
+}

src/machine/preprocessor.rs

@@ -0,0 +1,987 @@
+use crate::prolog_parser::ast::*;
+use crate::prolog_parser::tabled_rc::*;
+
+use crate::forms::*;
+use crate::iterators::*;
+use crate::machine::*;
+use crate::machine::load_state::*;
+use crate::machine::machine_errors::*;
+
+use crate::indexmap::IndexSet;
+
+use std::cell::Cell;
+use std::collections::VecDeque;
+use std::convert::TryFrom;
+use std::rc::Rc;
+
+/*
+ *  The preprocessor fabricates if-then-else ( .. -> ... ; ...)
+ *  clauses into nameless standalone predicates, which it queues for
+ *  later preprocessing and compilation. Fabricated predicates inherit
+ *  explicit "cut variables" from the handwritten predicate
+ *  surrounding their source if-then-else. They must be specially
+ *  handled.
+ */
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum CutContext {
+    BlocksCuts,
+    HasCutVariable,
+}
+
+pub fn fold_by_str<I>(terms: I, mut term: Term, sym: ClauseName) -> Term
+where
+    I: DoubleEndedIterator<Item = Term>,
+{
+    for prec in terms.rev() {
+        term = Term::Clause(
+            Cell::default(),
+            sym.clone(),
+            vec![Box::new(prec), Box::new(term)],
+            None,
+        );
+    }
+
+    term
+}
+
+pub fn to_op_decl(prec: usize, spec: &str, name: ClauseName) -> Result<OpDecl, CompilationError> {
+    match spec {
+        "xfx" => Ok(OpDecl::new(prec, XFX, name)),
+        "xfy" => Ok(OpDecl::new(prec, XFY, name)),
+        "yfx" => Ok(OpDecl::new(prec, YFX, name)),
+        "fx" => Ok(OpDecl::new(prec, FX, name)),
+        "fy" => Ok(OpDecl::new(prec, FY, name)),
+        "xf" => Ok(OpDecl::new(prec, XF, name)),
+        "yf" => Ok(OpDecl::new(prec, YF, name)),
+        _ => Err(CompilationError::InconsistentEntry),
+    }
+}
+
+fn setup_op_decl(
+    mut terms: Vec<Box<Term>>,
+    atom_tbl: TabledData<Atom>,
+) -> Result<OpDecl, CompilationError> {
+    let name = match *terms.pop().unwrap() {
+        Term::Constant(_, Constant::Atom(name, _)) => name,
+        Term::Constant(_, Constant::Char(c)) => clause_name!(c.to_string(), atom_tbl),
+        _ => return Err(CompilationError::InconsistentEntry),
+    };
+
+    let spec = match *terms.pop().unwrap() {
+        Term::Constant(_, Constant::Atom(name, _)) => name,
+        Term::Constant(_, Constant::Char(c)) => clause_name!(c.to_string(), atom_tbl),
+        _ => return Err(CompilationError::InconsistentEntry),
+    };
+
+    let prec = match *terms.pop().unwrap() {
+        Term::Constant(_, Constant::Fixnum(bi)) => match usize::try_from(bi) {
+            Ok(n) if n <= 1200 => n,
+            _ => return Err(CompilationError::InconsistentEntry),
+        },
+        _ => return Err(CompilationError::InconsistentEntry),
+    };
+
+    to_op_decl(prec, spec.as_str(), name)
+}
+
+fn setup_predicate_indicator(term: &mut Term) -> Result<PredicateKey, CompilationError>
+{
+    match term {
+        Term::Clause(_, ref slash, ref mut terms, Some(_))
+            if (slash.as_str() == "/" || slash.as_str() == "//") && terms.len() == 2 =>
+        {
+            let arity = *terms.pop().unwrap();
+            let name  = *terms.pop().unwrap();
+
+            let arity = arity
+                .to_constant()
+                .and_then(|c| {
+                    match c {
+                        Constant::Integer(n) => n.to_usize(),
+                        Constant::Fixnum(n) => usize::try_from(n).ok(),
+                        _ => None
+                    }
+                })
+                .ok_or(CompilationError::InvalidModuleExport)?;
+
+            let name  = name
+                .to_constant()
+                .and_then(|c| c.to_atom())
+                .ok_or(CompilationError::InvalidModuleExport)?;
+
+            if slash.as_str() == "/" {
+                Ok((name, arity))
+            } else {
+                Ok((name, arity + 2))
+            }
+        }
+        _ => {
+            Err(CompilationError::InvalidModuleExport)
+        }
+    }
+}
+
+/*
+fn setup_scoped_predicate_indicator(term: &mut Term) -> Result<ScopedPredicateKey, CompilationError>
+{
+    match term {
+        Term::Clause(_, ref name, ref mut terms, Some(_))
+            if name.as_str() == ":" && terms.len() == 2 =>
+        {
+            let mut predicate_indicator = *terms.pop().unwrap();
+            let module_name = *terms.pop().unwrap();
+
+            let module_name = module_name
+                .to_constant()
+                .and_then(|c| c.to_atom())
+                .ok_or(CompilationError::InvalidModuleExport)?;
+
+            let key = setup_predicate_indicator(&mut predicate_indicator)?;
+
+            Ok((module_name, key))
+        }
+        _ => Err(CompilationError::InvalidModuleExport),
+    }
+}
+*/
+
+fn setup_module_export(
+    mut term: Term,
+    atom_tbl: TabledData<Atom>,
+) -> Result<ModuleExport, CompilationError> {
+    setup_predicate_indicator(&mut term)
+        .map(ModuleExport::PredicateKey)
+        .or_else(|_| {
+            if let Term::Clause(_, name, terms, _) = term {
+                if terms.len() == 3 && name.as_str() == "op" {
+                    Ok(ModuleExport::OpDecl(setup_op_decl(
+                        terms,
+                        atom_tbl
+                    )?))
+                } else {
+                    Err(CompilationError::InvalidModuleDecl)
+                }
+            } else {
+                Err(CompilationError::InvalidModuleDecl)
+            }
+        })
+}
+
+pub(super)
+fn setup_module_export_list(
+    mut export_list: Term,
+    atom_tbl: TabledData<Atom>,
+) -> Result<Vec<ModuleExport>, CompilationError> {
+    let mut exports = vec![];
+
+    while let Term::Cons(_, t1, t2) = export_list {
+        let module_export = setup_module_export(*t1, atom_tbl.clone())?;
+
+        exports.push(module_export);
+        export_list = *t2;
+    }
+
+    if export_list.to_constant() != Some(Constant::EmptyList) {
+        Err(CompilationError::InvalidModuleDecl)
+    } else {
+        Ok(exports)
+    }
+}
+
+fn setup_module_decl(
+    mut terms: Vec<Box<Term>>,
+    atom_tbl: TabledData<Atom>,
+) -> Result<ModuleDecl, CompilationError> {
+    let export_list = *terms.pop().unwrap();
+    let name = terms
+        .pop()
+        .unwrap()
+        .to_constant()
+        .and_then(|c| c.to_atom())
+        .ok_or(CompilationError::InvalidModuleDecl)?;
+
+    let exports = setup_module_export_list(export_list, atom_tbl)?;
+    Ok(ModuleDecl { name, exports })
+}
+
+fn setup_use_module_decl(mut terms: Vec<Box<Term>>) -> Result<ModuleSource, CompilationError> {
+    match *terms.pop().unwrap() {
+        Term::Clause(_, ref name, ref mut terms, None)
+            if name.as_str() == "library" && terms.len() == 1 =>
+        {
+            terms
+                .pop()
+                .unwrap()
+                .to_constant()
+                .and_then(|c| c.to_atom())
+                .map(|c| ModuleSource::Library(c))
+                .ok_or(CompilationError::InvalidUseModuleDecl)
+        }
+        Term::Constant(_, Constant::Atom(ref name, _)) =>
+            Ok(ModuleSource::File(name.clone())),
+        _ => Err(CompilationError::InvalidUseModuleDecl),
+    }
+}
+
+/*
+fn setup_double_quotes(mut terms: Vec<Box<Term>>) -> Result<DoubleQuotes, CompilationError> {
+    let dbl_quotes = *terms.pop().unwrap();
+
+    match terms[0].as_ref() {
+        Term::Constant(_, Constant::Atom(ref name, _))
+            if name.as_str() == "double_quotes" => {
+                match dbl_quotes {
+                    Term::Constant(_, Constant::Atom(name, _)) => {
+                        match name.as_str() {
+                            "atom"  => Ok(DoubleQuotes::Atom),
+                            "chars" => Ok(DoubleQuotes::Chars),
+                            "codes" => Ok(DoubleQuotes::Codes),
+                            _ => Err(CompilationError::InvalidDoubleQuotesDecl),
+                        }
+                    }
+                    _ => {
+                        Err(CompilationError::InvalidDoubleQuotesDecl)
+                    }
+                }
+            },
+        _ => {
+            Err(CompilationError::InvalidDoubleQuotesDecl)
+        }
+    }
+}
+ */
+
+type UseModuleExport = (ModuleSource, IndexSet<ModuleExport>);
+
+fn setup_qualified_import(
+    mut terms: Vec<Box<Term>>,
+    atom_tbl: TabledData<Atom>,
+) -> Result<UseModuleExport, CompilationError> {
+    let mut export_list = *terms.pop().unwrap();
+    let module_src = match *terms.pop().unwrap() {
+        Term::Clause(_, ref name, ref mut terms, None)
+            if name.as_str() == "library" && terms.len() == 1 =>
+        {
+            terms
+                .pop()
+                .unwrap()
+                .to_constant()
+                .and_then(|c| c.to_atom())
+                .map(|c| ModuleSource::Library(c))
+                .ok_or(CompilationError::InvalidUseModuleDecl)
+        }
+        Term::Constant(_, Constant::Atom(ref name, _)) => {
+            Ok(ModuleSource::File(name.clone()))
+        }
+        _ => {
+            Err(CompilationError::InvalidUseModuleDecl)
+        }
+    }?;
+
+    let mut exports = IndexSet::new();
+
+    while let Term::Cons(_, t1, t2) = export_list {
+        exports.insert(setup_module_export(*t1, atom_tbl.clone())?);
+        export_list = *t2;
+    }
+
+    if export_list.to_constant() != Some(Constant::EmptyList) {
+        Err(CompilationError::InvalidModuleDecl)
+    } else {
+        Ok((module_src, exports))
+    }
+}
+
+/*
+ * setup_meta_predicate tries to extract meta-predicate information
+ * from an appropriately formed declaration
+ *
+ * :- meta_predicate maplist(:, ?, ?).
+ *
+ * indicating that, for each QueryTerm call to maplist/3, the first
+ * argument is to be expanded with the call resolution ((:)/2)
+ * operator, the first argument of which is the name of the host
+ * module, as an atom. For example,
+ *
+ * p(X) :- maplist(X, [a,b,c], Result).
+ *
+ * If p/2 is defined in a module named "mod", the call is expanded to
+ *
+ * maplist(mod:X, [a,b,c], Result).
+ *
+ * before the predicate is compiled to WAM instructions.
+ *
+ * If the term bound to X -- the predicate to be called -- is
+ * qualified with (:)/2 already, the innermost qualifier is used for
+ * call resolution.
+ *
+ * The three arguments returned by a successful call are the module name,
+ * predicate name, and the list of meta-specs, one for each predicate argument.
+ *
+ * The module name might be used to specify intra-module meta-predicates whose
+ * module is not yet defined. There are several examples of this
+ * contained in src/lib/ops_and_meta_predicates.pl, which is loaded before
+ * src/lib/builtins.pl.
+ *
+ * Meta-specs have three forms:
+ *
+ * (:)  (the argument should be expanded with (:)/2 as described above)
+ * +    (mode declarations under the mode syntax, which currently have no effect)
+ * -
+ * ?
+ */
+fn setup_meta_predicate<'a>(
+    mut terms: Vec<Box<Term>>,
+    load_state: &LoadState<'a>,
+) -> Result<(ClauseName, ClauseName, Vec<MetaSpec>), CompilationError>
+{
+    fn get_name_and_meta_specs(
+        name: ClauseName,
+        terms: &mut [Box<Term>],
+    ) -> Result<(ClauseName, Vec<MetaSpec>), CompilationError> {
+        let mut meta_specs = vec![];
+
+        for meta_spec in terms.into_iter() {
+            match &**meta_spec {
+                Term::Constant(_, Constant::Atom(meta_spec, _)) => {
+                    let meta_spec =
+                        match meta_spec.as_str() {
+                            ":" => MetaSpec::RequiresExpansion,
+                            "+" => MetaSpec::Plus,
+                            "-" => MetaSpec::Minus,
+                            "?" => MetaSpec::Either,
+                            _   => return Err(CompilationError::InvalidMetaPredicateDecl),
+                        };
+
+                    meta_specs.push(meta_spec);
+                }
+                Term::Constant(_, Constant::Fixnum(n)) => {
+                    match usize::try_from(*n) {
+                        Ok(n) if n <= MAX_ARITY => {
+                            meta_specs.push(MetaSpec::RequiresExpansionWithArgument(n));
+                        }
+                        _ => {
+                            return Err(CompilationError::InvalidMetaPredicateDecl);
+                        }
+                    }
+                }
+                _ => {
+                    return Err(CompilationError::InvalidMetaPredicateDecl);
+                }
+            }
+        }
+
+        Ok((name, meta_specs))
+    }
+
+    match *terms.pop().unwrap() {
+        Term::Clause(_, name, mut terms, _)
+            if name.as_str() == ":" && terms.len() == 2 => {
+                let spec = *terms.pop().unwrap();
+                let module_name = *terms.pop().unwrap();
+
+                match module_name {
+                    Term::Constant(_, Constant::Atom(module_name, _)) => {
+                        match spec {
+                            Term::Clause(_, name, mut terms, _) => {
+                                let (name, meta_specs) =
+                                    get_name_and_meta_specs(name, &mut terms)?;
+
+                                Ok((module_name, name, meta_specs))
+                            }
+                            _ => {
+                                Err(CompilationError::InvalidMetaPredicateDecl)
+                            }
+                        }
+                    }
+                    _ => {
+                        Err(CompilationError::InvalidMetaPredicateDecl)
+                    }
+                }
+            }
+        Term::Clause(_, name, mut terms, _) => {
+            let (name, meta_specs) = get_name_and_meta_specs(name, &mut terms)?;
+            Ok((load_state.module_name(), name, meta_specs))
+        }
+        _ => {
+            Err(CompilationError::InvalidMetaPredicateDecl)
+        }
+    }
+}
+
+fn merge_clauses(tls: &mut VecDeque<TopLevel>) -> Result<TopLevel, CompilationError>
+{
+    let mut clauses = vec![];
+
+    while let Some(tl) = tls.pop_front() {
+        match tl {
+            TopLevel::Query(_) if clauses.is_empty() && tls.is_empty() => {
+                return Ok(tl);
+            }
+            TopLevel::Declaration(_) if clauses.is_empty() => {
+                return Ok(tl);
+            }
+            TopLevel::Query(_) => {
+                return Err(CompilationError::InconsistentEntry);
+            }
+            TopLevel::Fact(fact) => {
+                let clause = PredicateClause::Fact(fact);
+                clauses.push(clause);
+            }
+            TopLevel::Rule(rule) => {
+                let clause = PredicateClause::Rule(rule);
+                clauses.push(clause);
+            }
+            TopLevel::Predicate(predicate) => {
+                clauses.extend(predicate.into_iter())
+            }
+            _ => {
+                tls.push_front(tl);
+                break;
+            }
+        }
+    }
+
+    if clauses.is_empty() {
+        Err(CompilationError::InconsistentEntry)
+    } else {
+        Ok(TopLevel::Predicate(clauses))
+    }
+}
+
+fn mark_cut_variables_as(terms: &mut Vec<Term>, name: ClauseName) {
+    for term in terms.iter_mut() {
+        match term {
+            &mut Term::Constant(_, Constant::Atom(ref mut var, _)) if var.as_str() == "!" => {
+                *var = name.clone()
+            }
+            _ => {}
+        }
+    }
+}
+
+fn mark_cut_variable(term: &mut Term) -> bool {
+    let cut_var_found = match term {
+        &mut Term::Constant(_, Constant::Atom(ref var, _)) if var.as_str() == "!" => true,
+        _ => false,
+    };
+
+    if cut_var_found {
+        *term = Term::Var(Cell::default(), rc_atom!("!"));
+        true
+    } else {
+        false
+    }
+}
+
+fn mark_cut_variables(terms: &mut Vec<Term>) -> bool {
+    let mut found_cut_var = false;
+
+    for item in terms.iter_mut() {
+        found_cut_var = mark_cut_variable(item) || found_cut_var;
+    }
+
+    found_cut_var
+}
+
+// terms is a list of goals composing one clause in a (;) functor. it
+// checks that the first (and only) of these clauses is a ->. if so,
+// it expands its terms using a blocked_!.
+fn check_for_internal_if_then(terms: &mut Vec<Term>) {
+    if terms.len() != 1 {
+        return;
+    }
+
+    if let Some(Term::Clause(_, ref name, ref subterms, _)) = terms.last() {
+        if name.as_str() != "->" || subterms.len() != 2 {
+            return;
+        }
+    } else {
+        return;
+    }
+
+    if let Some(Term::Clause(_, _, mut subterms, _)) = terms.pop() {
+        let mut conq_terms = VecDeque::from(unfold_by_str(*subterms.pop().unwrap(), ","));
+        let mut pre_cut_terms = VecDeque::from(unfold_by_str(*subterms.pop().unwrap(), ","));
+
+        conq_terms.push_front(Term::Constant(
+            Cell::default(),
+            Constant::Atom(clause_name!("blocked_!"), None))
+        );
+
+        while let Some(term) = pre_cut_terms.pop_back() {
+            conq_terms.push_front(term);
+        }
+
+        let tail_term = conq_terms.pop_back().unwrap();
+
+        terms.push(fold_by_str(
+            conq_terms.into_iter(),
+            tail_term,
+            clause_name!(","),
+        ));
+    }
+}
+
+fn setup_declaration<'a>(
+    load_state: &LoadState<'a>,
+    mut terms: Vec<Box<Term>>,
+) -> Result<Declaration, CompilationError> {
+    let term = *terms.pop().unwrap();
+    let atom_tbl = load_state.wam.machine_st.atom_tbl.clone();
+
+    match term {
+        Term::Clause(_, name, mut terms, _) =>
+            match (name.as_str(), terms.len()) {
+                ("dynamic", 1) => {
+		            let (name, arity) = setup_predicate_indicator(&mut *terms.pop().unwrap())?;
+		            Ok(Declaration::Dynamic(name, arity))
+		        }
+                ("module", 2) =>
+                    Ok(Declaration::Module(setup_module_decl(terms, atom_tbl)?)),
+                ("op", 3) =>
+                    Ok(Declaration::Op(setup_op_decl(terms, atom_tbl)?)),
+                ("non_counted_backtracking", 1) => {
+                    let (name, arity) = setup_predicate_indicator(&mut *terms.pop().unwrap())?;
+                    Ok(Declaration::NonCountedBacktracking(name, arity))
+                }
+                ("use_module", 1) => {
+                    Ok(Declaration::UseModule(setup_use_module_decl(terms)?))
+                }
+                ("use_module", 2) => {
+                    let (name, exports) = setup_qualified_import(terms, atom_tbl)?;
+                    Ok(Declaration::UseQualifiedModule(name, exports))
+                }
+                ("meta_predicate", 1) => {
+                    let (module_name, name, meta_specs) = setup_meta_predicate(terms, load_state)?;
+                    Ok(Declaration::MetaPredicate(module_name, name, meta_specs))
+                }
+                _ => {
+                    Err(CompilationError::InconsistentEntry)
+                }
+            },
+        _ => {
+            Err(CompilationError::InconsistentEntry)
+        }
+    }
+}
+
+#[inline]
+fn clause_to_query_term<'a>(
+    load_state: &mut LoadState<'a>,
+    name: ClauseName,
+    terms: Vec<Box<Term>>,
+    fixity: Option<SharedOpDesc>,
+) -> QueryTerm {
+    let ct = load_state.get_clause_type(name, terms.len(), fixity);
+    QueryTerm::Clause(Cell::default(), ct, terms, false)
+}
+
+#[inline]
+fn qualified_clause_to_query_term<'a>(
+    load_state: &mut LoadState<'a>,
+    module_name: ClauseName,
+    name: ClauseName,
+    terms: Vec<Box<Term>>,
+    fixity: Option<SharedOpDesc>,
+) -> QueryTerm {
+    let ct = load_state.get_qualified_clause_type(module_name, name, terms.len(), fixity);
+    QueryTerm::Clause(Cell::default(), ct, terms, false)
+}
+
+#[derive(Debug)]
+pub(crate) struct Preprocessor {
+    flags: MachineFlags,
+    queue: VecDeque<VecDeque<Term>>,
+}
+
+impl Preprocessor {
+    pub(super)
+    fn new(flags: MachineFlags) -> Self {
+        Preprocessor {
+            flags,
+            queue: VecDeque::new(),
+        }
+    }
+
+    fn setup_fact(&mut self, term: Term) -> Result<Term, CompilationError> {
+        match term {
+            Term::Clause(..) | Term::Constant(_, Constant::Atom(..)) => {
+                Ok(term)
+            }
+            _ => {
+                Err(CompilationError::InadmissibleFact)
+            }
+        }
+    }
+
+    fn compute_head(&self, term: &Term) -> Vec<Term> {
+        let mut vars = IndexSet::new();
+
+        for term in post_order_iter(term) {
+            if let TermRef::Var(_, _, v) = term {
+                vars.insert(v.clone());
+            }
+        }
+
+        vars.insert(rc_atom!("!"));
+        vars.into_iter()
+            .map(|v| Term::Var(Cell::default(), v))
+            .collect()
+    }
+
+    fn fabricate_rule_body(&self, vars: &Vec<Term>, body_term: Term) -> Term {
+        let vars_of_head = vars.iter().cloned().map(Box::new).collect();
+        let head_term = Term::Clause(Cell::default(), clause_name!(""), vars_of_head, None);
+
+        let rule = vec![Box::new(head_term), Box::new(body_term)];
+        let turnstile = clause_name!(":-");
+
+        Term::Clause(Cell::default(), turnstile, rule, None)
+    }
+
+    // the terms form the body of the rule. We create a head, by
+    // gathering variables from the body of terms and recording them
+    // in the head clause.
+    fn fabricate_rule(&self, body_term: Term) -> (JumpStub, VecDeque<Term>) {
+        // collect the vars of body_term into a head, return the num_vars
+        // (the arity) as well.
+        let vars = self.compute_head(&body_term);
+        let rule = self.fabricate_rule_body(&vars, body_term);
+
+        (vars, VecDeque::from(vec![rule]))
+    }
+
+    fn fabricate_disjunct(&self, body_term: Term) -> (JumpStub, VecDeque<Term>) {
+        let vars = self.compute_head(&body_term);
+        let results = unfold_by_str(body_term, ";")
+            .into_iter()
+            .map(|term| {
+                let mut subterms = unfold_by_str(term, ",");
+                mark_cut_variables(&mut subterms);
+
+                check_for_internal_if_then(&mut subterms);
+
+                let term = subterms.pop().unwrap();
+                let clause = fold_by_str(subterms.into_iter(), term, clause_name!(","));
+
+                self.fabricate_rule_body(&vars, clause)
+            })
+            .collect();
+
+        (vars, results)
+    }
+
+    fn fabricate_if_then(&self, prec: Term, conq: Term) -> (JumpStub, VecDeque<Term>) {
+        let mut prec_seq = unfold_by_str(prec, ",");
+        let comma_sym = clause_name!(",");
+        let cut_sym = atom!("!");
+
+        prec_seq.push(Term::Constant(Cell::default(), cut_sym));
+
+        mark_cut_variables_as(&mut prec_seq, clause_name!("blocked_!"));
+
+        let mut conq_seq = unfold_by_str(conq, ",");
+
+        mark_cut_variables(&mut conq_seq);
+        prec_seq.extend(conq_seq.into_iter());
+
+        let back_term = Box::new(prec_seq.pop().unwrap());
+        let front_term = Box::new(prec_seq.pop().unwrap());
+
+        let body_term = Term::Clause(
+            Cell::default(),
+            comma_sym.clone(),
+            vec![front_term, back_term],
+            None,
+        );
+
+        self.fabricate_rule(fold_by_str(prec_seq.into_iter(), body_term, comma_sym))
+    }
+
+    fn to_query_term<'a>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+        term: Term,
+    ) -> Result<QueryTerm, CompilationError> {
+        match term {
+            Term::Constant(_, Constant::Atom(name, fixity)) => {
+                if name.as_str() == "!" || name.as_str() == "blocked_!" {
+                    Ok(QueryTerm::BlockedCut)
+                } else {
+                    Ok(clause_to_query_term(load_state, name, vec![], fixity))
+                }
+            }
+            Term::Var(_, ref v) if v.as_str() == "!" => {
+                Ok(QueryTerm::UnblockedCut(Cell::default()))
+            }
+            Term::Clause(r, name, mut terms, fixity) => {
+                match (name.as_str(), terms.len()) {
+                    (";", 2) => {
+                        let term = Term::Clause(r, name.clone(), terms, fixity);
+
+                        let (stub, clauses) = self.fabricate_disjunct(term);
+                        self.queue.push_back(clauses);
+
+                        Ok(QueryTerm::Jump(stub))
+                    }
+                    ("->", 2) => {
+                        let conq = *terms.pop().unwrap();
+                        let prec = *terms.pop().unwrap();
+
+                        let (stub, clauses) = self.fabricate_if_then(prec, conq);
+                        self.queue.push_back(clauses);
+
+                        Ok(QueryTerm::Jump(stub))
+                    }
+                    ("\\+", 1) => {
+                        terms.push(Box::new(Term::Constant(
+                            Cell::default(),
+                            Constant::Atom(clause_name!("$fail"), None)
+                        )));
+
+                        let conq = Term::Constant(
+                            Cell::default(),
+                            Constant::Atom(clause_name!("true"), None)
+                        );
+
+                        let prec = Term::Clause(Cell::default(), clause_name!("->"), terms, None);
+                        let terms = vec![Box::new(prec), Box::new(conq)];
+
+                        let term = Term::Clause(Cell::default(), clause_name!(";"), terms, None);
+                        let (stub, clauses) = self.fabricate_disjunct(term);
+
+                        debug_assert!(clauses.len() > 0);
+                        self.queue.push_back(clauses);
+
+                        Ok(QueryTerm::Jump(stub))
+                    }
+                    ("$get_level", 1) => {
+                        if let Term::Var(_, ref var) = *terms[0] {
+                            Ok(QueryTerm::GetLevelAndUnify(Cell::default(), var.clone()))
+                        } else {
+                            Err(CompilationError::InadmissibleQueryTerm)
+                        }
+                    }
+                    (":", 2) => {
+                        let predicate_name = *terms.pop().unwrap();
+                        let module_name = *terms.pop().unwrap();
+
+                        match (module_name, predicate_name) {
+                            (Term::Constant(_, Constant::Atom(module_name, _)),
+                             Term::Constant(_, Constant::Atom(predicate_name, fixity))) => {
+                                Ok(qualified_clause_to_query_term(
+                                    load_state,
+                                    module_name,
+                                    predicate_name,
+                                    vec![],
+                                    fixity,
+                                ))
+                            }
+                            (Term::Constant(_, Constant::Atom(module_name, _)),
+                             Term::Clause(_, name, terms, fixity)) => {
+                                Ok(qualified_clause_to_query_term(
+                                    load_state,
+                                    module_name,
+                                    name,
+                                    terms,
+                                    fixity,
+                                ))
+                            }
+                            (module_name, predicate_name) => {
+                                terms.push(Box::new(module_name));
+                                terms.push(Box::new(predicate_name));
+
+                                Ok(clause_to_query_term(load_state, name, terms, fixity))
+                            }
+                        }
+                    }
+                    _ => {
+                        Ok(clause_to_query_term(load_state, name, terms, fixity))
+                    }
+                }
+            }
+            Term::Var(..) => {
+                Ok(QueryTerm::Clause(
+                    Cell::default(),
+                    ClauseType::CallN,
+                    vec![Box::new(term)],
+                    false,
+                ))
+            }
+            _ => {
+                Err(CompilationError::InadmissibleQueryTerm)
+            }
+        }
+    }
+
+    fn pre_query_term<'a>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+        term: Term,
+    ) -> Result<QueryTerm, CompilationError> {
+        match term {
+            Term::Clause(r, name, mut subterms, fixity) => {
+                if subterms.len() == 1 && name.as_str() == "$call_with_default_policy" {
+                    self.to_query_term(load_state, *subterms.pop().unwrap())
+                        .map(|mut query_term| {
+                            query_term.set_default_caller();
+                            query_term
+                        })
+                } else {
+                    self.to_query_term(load_state, Term::Clause(r, name, subterms, fixity))
+                }
+            }
+            _ => {
+                self.to_query_term(load_state, term)
+            }
+        }
+    }
+
+    fn setup_query<'a>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+        terms: Vec<Box<Term>>,
+        cut_context: CutContext,
+    ) -> Result<Vec<QueryTerm>, CompilationError> {
+        let mut query_terms = vec![];
+        let mut work_queue = VecDeque::from(terms);
+
+        while let Some(term) = work_queue.pop_front() {
+            let mut term = *term;
+
+            if let Term::Clause(cell, name, terms, op_spec) = term {
+                if name.as_str() == "," && terms.len() == 2 {
+                    let term = Term::Clause(cell, name, terms, op_spec);
+                    let mut subterms = unfold_by_str(term, ",");
+
+                    while let Some(subterm) = subterms.pop() {
+                        work_queue.push_front(Box::new(subterm));
+                    }
+
+                    continue;
+                } else {
+                    term = Term::Clause(cell, name, terms, op_spec);
+                }
+            }
+
+            if let CutContext::HasCutVariable = cut_context {
+                mark_cut_variable(&mut term);
+            }
+
+            query_terms.push(self.pre_query_term(load_state, term)?);
+        }
+
+        Ok(query_terms)
+    }
+
+    fn setup_rule<'a>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+        mut terms: Vec<Box<Term>>,
+        cut_context: CutContext,
+    ) -> Result<Rule, CompilationError> {
+        let post_head_terms: Vec<_> = terms.drain(1 ..).collect();
+
+        let mut query_terms =
+            self.setup_query(load_state, post_head_terms, cut_context)?;
+
+        let clauses = query_terms.drain(1 ..).collect();
+        let qt = query_terms.pop().unwrap();
+
+        match *terms.pop().unwrap() {
+            Term::Clause(_, name, terms, _) => {
+                Ok(Rule {
+                    head: (name, terms, qt),
+                    clauses,
+                })
+            }
+            Term::Constant(_, Constant::Atom(name, _)) => {
+                Ok(Rule {
+                    head: (name, vec![], qt),
+                    clauses,
+                })
+            }
+            _ => {
+                Err(CompilationError::InvalidRuleHead)
+            }
+        }
+    }
+
+    fn try_term_to_query<'a>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+        terms: Vec<Box<Term>>,
+        cut_context: CutContext,
+    ) -> Result<TopLevel, CompilationError> {
+        Ok(TopLevel::Query(self.setup_query(load_state, terms, cut_context)?))
+    }
+
+    pub(super)
+    fn try_term_to_tl<'a>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+        term: Term,
+        cut_context: CutContext,
+    ) -> Result<TopLevel, CompilationError> {
+        match term {
+            Term::Clause(r, name, terms, fixity) => {
+                if name.as_str() == "?-" {
+                    self.try_term_to_query(load_state, terms, cut_context)
+                } else if name.as_str() == ":-" && terms.len() == 2 {
+                    Ok(TopLevel::Rule(self.setup_rule(
+                        load_state,
+                        terms,
+                        cut_context,
+                    )?))
+                } else if name.as_str() == ":-" && terms.len() == 1 {
+                    Ok(TopLevel::Declaration(setup_declaration(load_state, terms)?))
+                } else {
+                    let term = Term::Clause(r, name, terms, fixity);
+                    Ok(TopLevel::Fact(self.setup_fact(term)?))
+                }
+            }
+            term => {
+                Ok(TopLevel::Fact(self.setup_fact(term)?))
+            }
+        }
+    }
+
+    fn try_terms_to_tls<'a, I: IntoIterator<Item = Term>>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+        terms: I,
+        cut_context: CutContext,
+    ) -> Result<VecDeque<TopLevel>, CompilationError> {
+        let mut results = VecDeque::new();
+
+        for term in terms.into_iter() {
+            results.push_back(self.try_term_to_tl(load_state, term, cut_context)?);
+        }
+
+        Ok(results)
+    }
+
+    pub(super)
+    fn parse_queue<'a>(
+        &mut self,
+        load_state: &mut LoadState<'a>,
+    ) -> Result<VecDeque<TopLevel>, CompilationError> {
+        let mut queue = VecDeque::new();
+
+        while let Some(terms) = self.queue.pop_front() {
+            let clauses = merge_clauses(
+                &mut self.try_terms_to_tls(
+                    load_state,
+                    terms,
+                    CutContext::HasCutVariable,
+                )?
+            )?;
+
+            queue.push_back(clauses);
+        }
+
+        Ok(queue)
+    }
+}

src/machine/project_attributes.pl

@@ -1,11 +1,11 @@
 :- module('$project_atts', [copy_term/3]).
 
-'$attribute_goals_driver'(QueryVars, AttrVars) :-
-    gather_modules(AttrVars, Modules0, _),
+driver(QueryVars, AttrVars) :-
+    gather_attr_modules(AttrVars, Modules0),
     sort(Modules0, Modules),
     call_project_attributes(Modules, QueryVars, AttrVars),
-    call_attribute_goals(Modules, call_query_var_goals, QueryVars),
-    call_attribute_goals(Modules, call_attr_var_goals, AttrVars).
+    call_attribute_goals(Modules, '$project_atts':call_query_var_goals, QueryVars),
+    call_attribute_goals(Modules, '$project_atts':call_attr_var_goals, AttrVars).
 
 enqueue_goals(Goals0) :-
     nonvar(Goals0),
@@ -36,7 +36,7 @@ call_project_attributes([Module|Modules], QueryVars, AttrVars) :-
     call_project_attributes(Modules, QueryVars, AttrVars).
 
 call_attribute_goals([], _, _).
-call_attribute_goals([Module | Modules], GoalCaller, AttrVars) :-
+call_attribute_goals([Module|Modules], GoalCaller, AttrVars) :-
     call(GoalCaller, AttrVars, Module, Goals),
     enqueue_goals(Goals),
     call_attribute_goals(Modules, GoalCaller, AttrVars).
@@ -51,13 +51,13 @@ call_attribute_goals([Module | Modules], GoalCaller, AttrVars) :-
 
 call_query_var_goals([], _, []).
 call_query_var_goals([AttrVar|AttrVars], Module, Goals) :-
-    (  catch((  Module:attribute_goals(AttrVar, Goals, RGoals0)
-	     ,  atts:'$default_attr_list'(Module, AttrVar, RGoals0, RGoals)
-	     ),
-	     E,
-	     (  '$print_attribute_goals_exception'(Module, E),
-		atts:'$default_attr_list'(Module, AttrVar, Goals, RGoals)
-	     ))
+    (  catch((  Module:attribute_goals(AttrVar, Goals, RGoals0),
+	            atts:'$default_attr_list'(Module, AttrVar, RGoals0, RGoals)
+	         ),
+	         E,
+	         (  '$project_atts':'$print_attribute_goals_exception'(Module, E),
+		        atts:'$default_attr_list'(Module, AttrVar, Goals, RGoals)
+	         ))
     -> true
     ;  atts:'$default_attr_list'(Module, AttrVar, Goals, RGoals)
     ),
@@ -66,25 +66,14 @@ call_query_var_goals([AttrVar|AttrVars], Module, Goals) :-
 call_attr_var_goals([], _, []).
 call_attr_var_goals([AttrVar|AttrVars], Module, Goals) :-
     (  catch(Module:attribute_goals(AttrVar, Goals, RGoals),
-	     E,
-	     '$print_attribute_goals_exception'(Module, E)
-	     )
+	         E,
+	         '$project_atts':'$print_attribute_goals_exception'(Module, E)
+	        )
     -> true
     ;  true
     ),
     call_attr_var_goals(AttrVars, Module, RGoals).
 
-gather_modules([], [], _).
-gather_modules([AttrVar|AttrVars], Modules, Modules0) :-
-    '$get_attr_list'(AttrVar, Attrs),
-    gather_modules_for_attrs(Attrs, Modules, Modules0),
-    gather_modules(AttrVars, Modules0, _).
-
-gather_modules_for_attrs(Attrs, Modules, Modules) :-
-    var(Attrs), !.
-gather_modules_for_attrs([Attr|Attrs], [Module|Modules], Modules0) :-
-    '$module_of'(Module, Attr),
-    gather_modules_for_attrs(Attrs, Modules, Modules0).
 
 module_prefixed_goals([], _, Gs, Gs).
 module_prefixed_goals([G|Gs], Module, [MG|MGs], TailGs) :-
@@ -100,11 +89,25 @@ call_attribute_goals_with_module_prefix([Module | Modules], GoalCaller, AttrVars
     module_prefixed_goals(Goals0, Module, Goals, Gs),
     call_attribute_goals_with_module_prefix(Modules, GoalCaller, AttrVars, Gs).
 
+
+gather_attr_modules([], []).
+gather_attr_modules([AttrVar|AttrVars], Modules) :-
+    '$get_attr_list'(AttrVar, Attrs),
+    copy_attribute_modules(Attrs, Modules, Modules0),
+    gather_attr_modules(AttrVars, Modules0).
+
+copy_attribute_modules(Attrs, Ls, Ls) :-
+    var(Attrs), !.
+copy_attribute_modules([Module:_|Attrs], [Module|Modules0], Modules1) :-
+    copy_attribute_modules(Attrs, Modules0, Modules1).
+
+
 copy_term(Source, Dest, Goals) :-
-    '$term_attributed_variables'(Source, Vars),
-    gather_modules(Vars, Modules0, _),
+    '$term_attributed_variables'(Source, AttrVars),
+    gather_attr_modules(AttrVars, Modules0),
     sort(Modules0, Modules),
-    call_attribute_goals_with_module_prefix(Modules, call_query_var_goals, Vars, Goals0),
+    call_attribute_goals_with_module_prefix(Modules, '$project_atts':call_query_var_goals,
+                                            AttrVars, Goals0),
     sort(Goals0, Goals1),
     !,
     '$copy_term_without_attr_vars'([Source | Goals1], [Dest | Goals]).

src/machine/stack.rs

@@ -246,10 +246,6 @@ impl Stack {
         }
     }
 
-    pub fn take(&mut self) -> Self {
-        Stack { buf: self.buf.take(), _marker: PhantomData }
-    }
-
     #[inline]
     pub fn truncate(&mut self, b: usize) {
         if b == 0 {

src/machine/streams.rs

@@ -179,10 +179,10 @@ impl Drop for StreamInstance {
     fn drop(&mut self) {
         match self {
             StreamInstance::TcpStream(_, ref mut tcp_stream) => {
-                discard_result!(tcp_stream.shutdown(Shutdown::Both));
+                tcp_stream.shutdown(Shutdown::Both).unwrap();
             }
             StreamInstance::TlsStream(_, ref mut tls_stream) => {
-                discard_result!(tls_stream.shutdown());
+                tls_stream.shutdown().unwrap();
             }
             _ => {
             }

src/machine/system_calls.rs

@@ -6,14 +6,16 @@ use crate::clause_types::*;
 use crate::forms::*;
 use crate::heap_print::*;
 use crate::instructions::*;
+use crate::machine;
 use crate::machine::code_repo::CodeRepo;
 use crate::machine::copier::*;
 use crate::machine::code_walker::*;
 use crate::machine::machine_errors::*;
 use crate::machine::machine_indices::*;
 use crate::machine::machine_state::*;
+use crate::machine::preprocessor::to_op_decl;
 use crate::machine::streams::*;
-use crate::machine::toplevel::to_op_decl;
+
 use crate::ordered_float::OrderedFloat;
 use crate::read::readline;
 use crate::rug::Integer;
@@ -402,9 +404,7 @@ impl MachineState {
             }
         }
 
-        let mode =
-            atom_from!(self, indices, self.store(self.deref(self[temp_v!(2)])));
-
+        let mode = atom_from!(self, self.store(self.deref(self[temp_v!(2)])));
         let mut open_options = fs::OpenOptions::new();
 
         let (is_input_file, in_append_mode) =
@@ -560,7 +560,7 @@ impl MachineState {
                         let spec = get_clause_spec(
                             name.clone(),
                             *arity,
-                            composite_op!(&indices.op_dir),
+                            &CompositeOpDir::new(&indices.op_dir, None),
                         );
 
                         let addr = self.heap.to_unifiable(HeapCellValue::DBRef(
@@ -681,11 +681,11 @@ impl MachineState {
 
         let mut parser = Parser::new(
             &mut stream,
-            indices.atom_tbl.clone(),
+            self.atom_tbl.clone(),
             self.machine_flags(),
         );
 
-        match parser.read_term(composite_op!(&indices.op_dir)) {
+        match parser.read_term(&CompositeOpDir::new(&indices.op_dir, None)) {
             Err(err) => {
                 let h = self.heap.h();
                 let err = MachineError::syntax_error(h, err);
@@ -792,6 +792,7 @@ impl MachineState {
         current_output_stream: &mut Stream,
     ) -> CallResult {
         match ct {
+            /*
             &SystemClauseType::AbolishClause => {
                 let p = self.cp;
                 let trans_type = DynamicTransactionType::Abolish;
@@ -806,6 +807,7 @@ impl MachineState {
                 self.p = CodePtr::DynamicTransaction(trans_type, p);
                 return Ok(());
             }
+            */
             &SystemClauseType::BindFromRegister => {
                 let reg = self.store(self.deref(self[temp_v!(2)]));
                 let n =
@@ -833,6 +835,7 @@ impl MachineState {
 
                 self.fail = true;
             }
+            /*
             &SystemClauseType::AssertDynamicPredicateToFront => {
                 let p = self.cp;
                 let trans_type = DynamicTransactionType::Assert(DynamicAssertPlace::Front);
@@ -841,19 +844,21 @@ impl MachineState {
                 return Ok(());
             }
             &SystemClauseType::AssertDynamicPredicateToBack => {
-                let p = self.cp;
-                let trans_type = DynamicTransactionType::Assert(DynamicAssertPlace::Back);
+                // let p = self.cp;
+                // let trans_type = DynamicTransactionType::Assert(DynamicAssertPlace::Back);
 
-                self.p = CodePtr::DynamicTransaction(trans_type, p);
+                // self.p = CodePtr::DynamicTransaction(trans_type, p);
+                self.p = CodePtr::REPL(REPLCodePtr::UserAssertz, self.cp);
                 return Ok(());
             }
+            */
             &SystemClauseType::CurrentHostname => {
                 match hostname::get().ok() {
                     Some(host) => {
                         match host.into_string().ok() {
                             Some(host) => {
                                 let hostname = self.heap.to_unifiable(
-                                    HeapCellValue::Atom(clause_name!(host, indices.atom_tbl), None)
+                                    HeapCellValue::Atom(clause_name!(host, self.atom_tbl), None)
                                 );
 
                                 self.unify(self[temp_v!(1)], hostname);
@@ -1016,6 +1021,7 @@ impl MachineState {
                                    return Err(err);
                             }
                         };
+
                     let chars = self.heap.put_complete_string(current);
                     self.unify(self[temp_v!(1)], chars);
 
@@ -1092,11 +1098,13 @@ impl MachineState {
                     return Ok(());
                 }
             }
+            /*
             &SystemClauseType::AtEndOfExpansion => {
                 if self.cp == LocalCodePtr::TopLevel(0, 0) {
                     self.at_end_of_expansion = true;
                 }
             }
+            */
             &SystemClauseType::AtomChars => {
                 let a1 = self[temp_v!(1)];
 
@@ -1139,7 +1147,7 @@ impl MachineState {
                                 if &string == "[]" {
                                     self.unify(addr, Addr::EmptyList);
                                 } else {
-                                    let chars = clause_name!(string, indices.atom_tbl);
+                                    let chars = clause_name!(string, self.atom_tbl);
                                     let atom  = self.heap.to_unifiable(
                                         HeapCellValue::Atom(chars, None)
                                     );
@@ -1254,7 +1262,7 @@ impl MachineState {
                                 }
 
                                 let string = self.heap.to_unifiable(
-                                    HeapCellValue::Atom(clause_name!(chars, indices.atom_tbl), None)
+                                    HeapCellValue::Atom(clause_name!(chars, self.atom_tbl), None)
                                 );
 
                                 self.bind(addr.as_var().unwrap(), string);
@@ -1281,7 +1289,7 @@ impl MachineState {
                         clause_name!("[]")
                     }
                     Addr::Char(c) => {
-                        clause_name!(c.to_string(), indices.atom_tbl)
+                        clause_name!(c.to_string(), self.atom_tbl)
                     }
                     _ => {
                         unreachable!()
@@ -1858,6 +1866,7 @@ impl MachineState {
                     }
                 }
             }
+            /*
             &SystemClauseType::ModuleAssertDynamicPredicateToFront => {
                 let p = self.cp;
                 let trans_type = DynamicTransactionType::ModuleAssert(DynamicAssertPlace::Front);
@@ -1872,6 +1881,7 @@ impl MachineState {
                 self.p = CodePtr::DynamicTransaction(trans_type, p);
                 return Ok(());
             }
+            */
             &SystemClauseType::LiftedHeapLength => {
                 let a1 = self[temp_v!(1)];
                 let lh_len = Addr::Usize(self.lifted_heap.h());
@@ -2854,6 +2864,7 @@ impl MachineState {
 
                 self.unify(Addr::Char(c), a1);
             }
+/*
             &SystemClauseType::GetModuleClause => {
                 let module = self[temp_v!(3)];
                 let head = self[temp_v!(1)];
@@ -2944,20 +2955,24 @@ impl MachineState {
                     _ => unreachable!(),
                 };
             }
+*/
             &SystemClauseType::HeadIsDynamic => {
-                let head = self[temp_v!(1)];
+                let module_name = atom_from!(
+                    self,
+                    self.store(self.deref(
+                        self[temp_v!(1)]
+                    ))
+                );
 
-                self.fail = !match self.store(self.deref(head)) {
+                self.fail = !match self.store(self.deref(self[temp_v!(2)])) {
                     Addr::Str(s) => match &self.heap[s] {
-                        &HeapCellValue::NamedStr(arity, ref name, ..) => indices
-                            .get_clause_subsection(name.owning_module(), name.clone(), arity)
-                            .is_some(),
+                        &HeapCellValue::NamedStr(arity, ref name, ..) =>
+                            indices.is_dynamic_predicate(module_name, (name.clone(), arity)),
                         _ => unreachable!(),
                     },
                     Addr::Con(h) if self.heap.atom_at(h) => {
-	                if let HeapCellValue::Atom(name, _) = &self.heap[h] {
-                            indices.get_clause_subsection(name.owning_module(), name.clone(), 0)
-                                   .is_some()
+	                    if let HeapCellValue::Atom(name, _) = &self.heap[h] {
+                            indices.is_dynamic_predicate(module_name, (name.clone(), 0))
                         } else {
                             unreachable!()
                         }
@@ -3111,21 +3126,27 @@ impl MachineState {
 
                             return self.module_lookup(
                                 indices,
+                                call_policy,
                                 (name, arity + narity),
                                 module_name,
                                 true,
+                                current_input_stream,
+                                current_output_stream,
                             );
                         } else {
                             unreachable!()
                         }
                     }
                     Addr::Con(h) if self.heap.atom_at(h) => {
-	                if let HeapCellValue::Atom(name, _) = self.heap.clone(h) {
+	                    if let HeapCellValue::Atom(name, _) = self.heap.clone(h) {
                             return self.module_lookup(
                                 indices,
+                                call_policy,
                                 (name.clone(), narity),
                                 module_name,
                                 true,
+                                current_input_stream,
+                                current_output_stream,
                             );
                         } else {
                             unreachable!()
@@ -3160,6 +3181,7 @@ impl MachineState {
                     }
                 }
             }
+            /*
             &SystemClauseType::ExpandGoal => {
                 self.p = CodePtr::Local(LocalCodePtr::UserGoalExpansion(0));
                 return Ok(());
@@ -3168,6 +3190,7 @@ impl MachineState {
                 self.p = CodePtr::Local(LocalCodePtr::UserTermExpansion(0));
                 return Ok(());
             }
+            */
             &SystemClauseType::GetNextDBRef => {
                 let a1 = self[temp_v!(1)];
 
@@ -3185,7 +3208,7 @@ impl MachineState {
                             let spec = get_clause_spec(
                                 name.clone(),
                                 *arity,
-                                composite_op!(&indices.op_dir),
+                                &CompositeOpDir::new(&indices.op_dir, None),
                             );
 
                             let db_ref = DBRef::NamedPred(name.clone(), *arity, spec);
@@ -3429,7 +3452,7 @@ impl MachineState {
 
                 let op = match self.store(self.deref(op)) {
                     Addr::Char(c) =>
-                        clause_name!(c.to_string(), indices.atom_tbl),
+                        clause_name!(c.to_string(), self.atom_tbl),
                     Addr::Con(h) if self.heap.atom_at(h) =>
                         if let HeapCellValue::Atom(ref name, _) = &self.heap[h] {
                             name.clone()
@@ -3440,16 +3463,18 @@ impl MachineState {
                         unreachable!(),
                 };
 
-                let module = op.owning_module();
-
                 let result = to_op_decl(priority, specifier.as_str(), op)
                     .map_err(SessionError::from)
-                    .and_then(|op_decl| {
-                        if op_decl.0 == 0 {
+                    .and_then(|mut op_decl| {
+                        if op_decl.prec == 0 {
                             Ok(op_decl.remove(&mut indices.op_dir))
                         } else {
-                            let spec = get_desc(op_decl.name(), composite_op!(&indices.op_dir));
-                            op_decl.submit(module, spec, &mut indices.op_dir)
+                            let spec = get_op_desc(
+                                op_decl.name.clone(),
+                                &CompositeOpDir::new(&indices.op_dir, None),
+                            );
+
+                            op_decl.submit(spec, &mut indices.op_dir)
                         }
                     });
 
@@ -3493,10 +3518,10 @@ impl MachineState {
                                     let mut heap_pstr_iter =
                                         self.heap_pstr_iter(Addr::PStrLocation(h, n));
 
-                                    let file_spec = 
+                                    let file_spec =
                                         clause_name!(
                                             heap_pstr_iter.to_string(),
-                                            indices.atom_tbl.clone()
+                                            self.atom_tbl
                                         );
 
                                     self.stream_from_file_spec(file_spec, indices, &options)?
@@ -3901,65 +3926,18 @@ impl MachineState {
                     }
                 };
             }
-            &SystemClauseType::ModuleOf => {
-                let module = self.store(self.deref(self[temp_v!(2)]));
-
-                match module {
-                    Addr::Con(h) if self.heap.atom_at(h) => {
-	                if let HeapCellValue::Atom(name, _) = self.heap.clone(h) {
-                            let module = self.heap.to_unifiable(
-                                HeapCellValue::Atom(
-                                    name.owning_module(),
-                                    None
-                                ),
-                            );
-
-                            let target = self[temp_v!(1)];
-
-                            self.unify(target, module);
-                        } else {
-                            unreachable!()
-                        }
-                    }
-                    Addr::Str(s) => match self.heap.clone(s) {
-                        HeapCellValue::NamedStr(_, name, ..) => {
-                            let module = self.heap.to_unifiable(
-                                HeapCellValue::Atom(
-                                    name.owning_module(),
-                                    None
-                                ),
-                            );
-
-                            let target = self[temp_v!(1)];
-
-                            self.unify(target, module);
-                        }
-                        HeapCellValue::Addr(addr) if addr.is_ref() => {
-                            let err = MachineError::uninstantiation_error(addr);
-                            let stub = MachineError::functor_stub(
-                                clause_name!("$module_of"),
-                                2,
-                            );
-
-                            return Err(self.error_form(err, stub));
-                        }
-                        _ => {
-                            unreachable!()
-                        }
-                    },
-                    _ => {
-                        self.fail = true;
-                    }
-                };
-            }
             &SystemClauseType::NoSuchPredicate => {
-                let head = self[temp_v!(1)];
+                let module_name = atom_from!(self, self.store(self.deref(self[temp_v!(1)])));
 
-                self.fail = match self.store(self.deref(head)) {
+                self.fail = match self.store(self.deref(self[temp_v!(2)])) {
                     Addr::Str(s) => match &self.heap[s] {
                         &HeapCellValue::NamedStr(arity, ref name, ref spec) => {
-                            let module = name.owning_module();
-                            indices.predicate_exists(name.clone(), module, arity, spec.clone())
+                            indices.get_predicate_code_index(
+                                name.clone(),
+                                arity,
+                                module_name,
+                                spec.clone(),
+                            ).is_some()
                         }
                         _ => {
                             unreachable!()
@@ -3967,14 +3945,14 @@ impl MachineState {
                     },
                     Addr::Con(h) if self.heap.atom_at(h) => {
 	                    if let &HeapCellValue::Atom(ref name, ref spec) = &self.heap[h] {
-                            let module = name.owning_module();
                             let spec = fetch_atom_op_spec(
                                 name.clone(),
                                 spec.clone(),
                                 &indices.op_dir,
                             );
 
-                            indices.predicate_exists(name.clone(), module, 0, spec)
+                            indices.get_predicate_code_index(name.clone(), 0, module_name, spec)
+                                   .is_some()
                         } else {
                             unreachable!()
                         }
@@ -4116,6 +4094,7 @@ impl MachineState {
             &SystemClauseType::REPL(repl_code_ptr) => {
                 return self.repl_redirect(repl_code_ptr);
             }
+            /*
             &SystemClauseType::ModuleRetractClause => {
                 let p = self.cp;
                 let trans_type = DynamicTransactionType::ModuleRetract;
@@ -4130,6 +4109,7 @@ impl MachineState {
                 self.p = CodePtr::DynamicTransaction(trans_type, p);
                 return Ok(());
             }
+            */
             &SystemClauseType::ReturnFromVerifyAttr => {
                 let e = self.e;
                 let frame_len = self.stack.index_and_frame(e).prelude.univ_prelude.num_cells;
@@ -4326,6 +4306,7 @@ impl MachineState {
 
                 self.unify(a1, a2);
             }
+/*
             &SystemClauseType::GetClause => {
                 let head = self[temp_v!(1)];
 
@@ -4343,14 +4324,14 @@ impl MachineState {
                         }
                     },
                     Addr::Con(h) if self.heap.atom_at(h) => {
-	                if let &HeapCellValue::Atom(ref name, _) = &self.heap[h] {
+	                    if let &HeapCellValue::Atom(ref name, _) = &self.heap[h] {
                             indices.get_clause_subsection(
                                 name.owning_module(),
                                 name.clone(),
                                 0,
                             )
                         } else {
-                            unreachable!()
+                unreachable!()
                         }
                     }
                     _ => {
@@ -4372,6 +4353,7 @@ impl MachineState {
                     }
                 }
             }
+*/
             &SystemClauseType::GetCutPoint => {
                 let a1 = self[temp_v!(1)];
                 let a2 = Addr::CutPoint(self.b0);
@@ -4524,7 +4506,7 @@ impl MachineState {
                     let term_write_result =
                         match self.read(
                             Stream::from(chars),
-                            indices.atom_tbl.clone(),
+                            self.atom_tbl.clone(),
                             &indices.op_dir,
                         ) {
                             Ok(term_write_result) => {
@@ -4595,10 +4577,11 @@ impl MachineState {
                 let mut rand = RANDOM_STATE.borrow_mut();
                 rand.seed(&seed);
             }
-            &SystemClauseType::SkipMaxList =>
+            &SystemClauseType::SkipMaxList => {
                 if let Err(err) = self.skip_max_list() {
                     return Err(err);
-                },
+                }
+            }
             &SystemClauseType::Sleep => {
                 let time = self.store(self.deref(self[temp_v!(1)]));
 
@@ -4695,11 +4678,10 @@ impl MachineState {
                 let stream =
                     match TcpStream::connect(&socket_addr).map_err(|e| e.kind()) {
                         Ok(tcp_stream) => {
-                            let socket_addr = clause_name!(socket_addr, indices.atom_tbl.clone());
+                            let socket_addr = clause_name!(socket_addr, self.atom_tbl);
 
-
-                            let mut stream =
-                              { let tls = match self.store(self.deref(self[temp_v!(8)])) {
+                            let mut stream = {
+                                let tls = match self.store(self.deref(self[temp_v!(8)])) {
                                     Addr::Con(h) if self.heap.atom_at(h) => {
                                         if let HeapCellValue::Atom(ref atom, _) = &self.heap[h] {
                                             atom.as_str()
@@ -4711,6 +4693,7 @@ impl MachineState {
                                         unreachable!()
                                     }
                                 };
+
                                 match tls {
                                   "false" => { Stream::from_tcp_stream(socket_addr, tcp_stream) }
                                   "true" => { let connector = TlsConnector::new().unwrap();
@@ -4724,6 +4707,7 @@ impl MachineState {
                                    _ => { unreachable!() }
                                 }
                               };
+
                             stream.options = options;
 
                             if let Some(ref alias) = &stream.options.alias {
@@ -4877,7 +4861,7 @@ impl MachineState {
                                 match tcp_listener.accept().ok() {
                                     Some((tcp_stream, socket_addr)) => {
                                         let client =
-                                            clause_name!(format!("{}", socket_addr), indices.atom_tbl);
+                                            clause_name!(format!("{}", socket_addr), self.atom_tbl);
 
                                         let mut tcp_stream =
                                             Stream::from_tcp_stream(client.clone(), tcp_stream);
@@ -5226,8 +5210,13 @@ impl MachineState {
                 self.fail = self.structural_eq_test();
             }
             &SystemClauseType::WAMInstructions => {
-                let name  = self[temp_v!(1)];
-                let arity = self[temp_v!(2)];
+                let module_name = atom_from!(
+                    self,
+                    self.store(self.deref(self[temp_v!(1)]))
+                );
+
+                let name  = self[temp_v!(2)];
+                let arity = self[temp_v!(3)];
 
                 let name = match self.store(self.deref(name)) {
                     Addr::Con(h) if self.heap.atom_at(h) => {
@@ -5257,51 +5246,84 @@ impl MachineState {
                         }
                     };
 
-                let first_idx = match indices
-                    .code_dir
-                    .get(&(name.clone(), arity.to_usize().unwrap()))
-                {
-                    Some(ref idx) if idx.local().is_some() => {
-                        if let Some(idx) = idx.local() {
-                            idx
-                        } else {
-                            unreachable!()
+                let key = (name.clone(), arity.to_usize().unwrap());
+
+                let first_idx = match module_name.as_str() {
+                    "user" => indices.code_dir.get(&key),
+                    _ => match indices.modules.get(&module_name) {
+                        Some(module) => module.code_dir.get(&key),
+                        None => {
+                            let stub = MachineError::functor_stub(key.0, key.1);
+                            let h = self.heap.h();
+
+                            let err = MachineError::session_error(
+                                h,
+                                SessionError::from(
+                                    CompilationError::InvalidModuleResolution(
+                                        module_name
+                                    )
+                                ),
+                            );
+
+                            let err = self.error_form(err, stub);
+
+                            self.throw_exception(err);
+                            return Ok(());
                         }
-                    }
-                    _ => {
-                        let arity = arity.to_usize().unwrap();
-                        let stub = MachineError::functor_stub(name.clone(), arity);
-                        let h = self.heap.h();
-
-                        let err = MachineError::existence_error(
-                            h,
-                            ExistenceError::Procedure(name, arity),
-                        );
-
-                        let err = self.error_form(err, stub);
-
-                        self.throw_exception(err);
-                        return Ok(());
-                    }
+                    },
                 };
 
+                let first_idx =
+                    match first_idx {
+                        Some(ref idx) if idx.local().is_some() => {
+                            if let Some(idx) = idx.local() {
+                                idx
+                            } else {
+                                unreachable!()
+                            }
+                        }
+                        _ => {
+                            let arity = arity.to_usize().unwrap();
+                            let stub = MachineError::functor_stub(name.clone(), arity);
+                            let h = self.heap.h();
+
+                            let err = MachineError::existence_error(
+                                h,
+                                ExistenceError::Procedure(name, arity),
+                            );
+
+                            let err = self.error_form(err, stub);
+
+                            self.throw_exception(err);
+                            return Ok(());
+                        }
+                    };
+
                 let mut h = self.heap.h();
                 let mut functors = vec![];
+                let mut functor_list = vec![];
 
                 walk_code(
                     &code_repo.code,
                     first_idx,
                     |instr| {
-                        let section = instr.to_functor(h);
-                        functors.push(Addr::HeapCell(h));
+                        let old_len = functors.len();
+                        instr.enqueue_functors(h, &mut functors);
+                        let new_len = functors.len();
 
-                        h += section.len();
-                        self.heap.extend(section.into_iter());
+                        for index in old_len .. new_len {
+                            functor_list.push(Addr::HeapCell(h));
+                            h += functors[index].len();
+                        }
                     },
                 );
 
-                let listing = Addr::HeapCell(self.heap.to_list(functors.into_iter()));
-                let listing_var = self[temp_v!(3)];
+                for functor in functors {
+                    self.heap.extend(functor.into_iter());
+                }
+
+                let listing = Addr::HeapCell(self.heap.to_list(functor_list.into_iter()));
+                let listing_var = self[temp_v!(4)];
 
                 self.unify(listing, listing_var);
             }
@@ -5831,6 +5853,53 @@ impl MachineState {
                     self.unify(self[temp_v!(2)], cstr);
                 }
             }
+            &SystemClauseType::LoadLibraryAsStream => {
+                let library_name =
+                    atom_from!(
+                        self,
+                        self.store(self.deref(self[temp_v!(1)]))
+                    );
+
+                use crate::LIBRARIES;
+
+                match LIBRARIES.borrow().get(library_name.as_str()) {
+                    Some(library) => {
+                        let var_ref = Ref::HeapCell(self.heap.push(
+                            HeapCellValue::Stream(Stream::from(*library))
+                        ));
+
+                        self.bind(var_ref, self[temp_v!(2)]);
+
+                        let mut path_buf = machine::current_dir();
+
+                        path_buf.push("/lib");
+                        path_buf.push(library_name.as_str());
+
+                        let library_path_str = path_buf.to_str().unwrap();
+                        let library_path =
+                            clause_name!(library_path_str.to_string(), self.atom_tbl);
+
+                        let library_path_ref = Ref::HeapCell(
+                            self.heap.push(HeapCellValue::Atom(library_path, None))
+                        );
+
+                        self.bind(library_path_ref, self[temp_v!(3)]);
+                    }
+                    None => {
+                        return Err(
+                            self.error_form(
+                                MachineError::existence_error(
+                                    self.heap.h(),
+                                    ExistenceError::ModuleSource(
+                                        ModuleSource::Library(library_name)
+                                    ),
+                                ),
+                                MachineError::functor_stub(clause_name!("load"), 1),
+                            )
+                        );
+                    }
+                }
+            }
         };
 
         return_from_clause!(self.last_call, self)
@@ -5905,7 +5974,7 @@ impl MachineState {
         } else {
             let mut avec = Vec::new();
             for attr in node.attributes() {
-                let chars = clause_name!(String::from(attr.name()), indices.atom_tbl);
+                let chars = clause_name!(String::from(attr.name()), self.atom_tbl);
                 let name  = self.heap.to_unifiable(
                     HeapCellValue::Atom(chars, None)
                 );
@@ -5926,7 +5995,7 @@ impl MachineState {
             }
             let children = Addr::HeapCell(self.heap.to_list(cvec.into_iter()));
 
-            let chars = clause_name!(String::from(node.tag_name().name()), indices.atom_tbl);
+            let chars = clause_name!(String::from(node.tag_name().name()), self.atom_tbl);
             let tag  = self.heap.to_unifiable(
                 HeapCellValue::Atom(chars, None)
             );
@@ -5955,7 +6024,7 @@ impl MachineState {
             Some(name) => {
                 let mut avec = Vec::new();
                 for attr in node.attrs() {
-                    let chars = clause_name!(String::from(attr.0), indices.atom_tbl);
+                    let chars = clause_name!(String::from(attr.0), self.atom_tbl);
                     let name  = self.heap.to_unifiable(
                         HeapCellValue::Atom(chars, None)
                     );
@@ -5976,7 +6045,7 @@ impl MachineState {
                 }
                 let children = Addr::HeapCell(self.heap.to_list(cvec.into_iter()));
 
-                let chars = clause_name!(String::from(name), indices.atom_tbl);
+                let chars = clause_name!(String::from(name), self.atom_tbl);
                 let tag  = self.heap.to_unifiable(
                     HeapCellValue::Atom(chars, None)
                 );

src/machine/term_expansion.rs

@@ -1,394 +0,0 @@
-use crate::prolog_parser::ast::*;
-use crate::prolog_parser::parser::*;
-
-use crate::machine::machine_indices::HeapCellValue;
-use crate::machine::*;
-use crate::rug::ops::Pow;
-use crate::rug::Integer;
-
-use std::cell::Cell;
-use std::collections::VecDeque;
-use std::iter::Rev;
-use std::vec::IntoIter;
-
-pub fn fold_by_str<I>(terms: I, mut term: Term, sym: ClauseName) -> Term
-where
-    I: DoubleEndedIterator<Item = Term>,
-{
-    for prec in terms.rev() {
-        term = Term::Clause(
-            Cell::default(),
-            sym.clone(),
-            vec![Box::new(prec), Box::new(term)],
-            None,
-        );
-    }
-
-    term
-}
-
-fn extract_from_list(
-    head: Box<Term>,
-    tail: Box<Term>,
-) -> Result<Rev<IntoIter<Term>>, ParserError>
-{
-    let mut terms = vec![*head];
-    let mut tail = *tail;
-
-    while let Term::Cons(_, head, next_tail) = tail {
-        terms.push(*head);
-        tail = *next_tail;
-    }
-
-    if let Term::Constant(_, Constant::EmptyList) = tail {
-        Ok(terms.into_iter().rev())
-    } else {
-        Err(ParserError::ExpectedTopLevelTerm)
-    }
-}
-
-#[derive(Debug)]
-pub struct TermStream<'a> {
-    stack: Vec<Term>,
-    pub(crate) wam: &'a mut Machine,
-    parser: Parser<'a, Stream>,
-    pub(crate) flags: MachineFlags,
-    term_expansion_lens: (usize, usize),
-    goal_expansion_lens: (usize, usize),
-    top_level_terms: Vec<(Term, usize, usize)>, // term, line_num, col_num.
-}
-
-#[derive(Debug)]
-pub struct ExpansionAdditionResult {
-    term_expansion_additions: (Predicate, VecDeque<TopLevel>),
-    goal_expansion_additions: (Predicate, VecDeque<TopLevel>),
-}
-
-impl ExpansionAdditionResult {
-    pub fn take_term_expansions(&mut self) -> (Predicate, VecDeque<TopLevel>) {
-        let tes = mem::replace(&mut self.term_expansion_additions.0, Predicate::new());
-        let teqs = mem::replace(&mut self.term_expansion_additions.1, VecDeque::from(vec![]));
-
-        (tes, teqs)
-    }
-
-    pub fn take_goal_expansions(&mut self) -> (Predicate, VecDeque<TopLevel>) {
-        let ges = mem::replace(&mut self.goal_expansion_additions.0, Predicate::new());
-        let geqs = mem::replace(&mut self.goal_expansion_additions.1, VecDeque::from(vec![]));
-
-        (ges, geqs)
-    }
-}
-
-impl<'a> Drop for TermStream<'a> {
-    fn drop(&mut self) {
-        self.wam.indices.in_situ_code_dir.clear();
-        self.wam.indices.in_situ_module_dir.clear();
-
-        self.wam.code_repo.in_situ_code.clear();
-        discard_result!(self.rollback_expansion_code());
-    }
-}
-
-impl<'a> TermStream<'a> {
-    pub fn new(
-        src: &'a mut ParsingStream<Stream>,
-        atom_tbl: TabledData<Atom>,
-        flags: MachineFlags,
-        wam: &'a mut Machine,
-    ) -> Self {
-        TermStream {
-            stack: Vec::new(),
-            term_expansion_lens: wam
-                .code_repo
-                .term_dir_entry_len((clause_name!("term_expansion"), 2)),
-            goal_expansion_lens: wam
-                .code_repo
-                .term_dir_entry_len((clause_name!("goal_expansion"), 2)),
-            wam,
-            parser: Parser::new(src, atom_tbl, flags),
-            flags,
-            top_level_terms: vec![],
-        }
-    }
-
-    #[inline]
-    pub fn top_level_terms(&mut self) -> Vec<(Term, usize, usize)> {
-        mem::replace(&mut self.top_level_terms, vec![])
-    }
-
-    #[inline]
-    pub fn incr_expansion_lens(&mut self, hook: CompileTimeHook, len: usize, queue_len: usize) {
-        match hook {
-            CompileTimeHook::UserTermExpansion => {
-                self.term_expansion_lens.0 += len;
-                self.term_expansion_lens.1 += queue_len;
-            }
-            CompileTimeHook::UserGoalExpansion => {
-                self.goal_expansion_lens.0 += len;
-                self.goal_expansion_lens.1 += queue_len;
-            }
-            _ => {}
-        }
-    }
-
-    #[inline]
-    pub fn line_num(&self) -> usize {
-        self.parser.line_num()
-    }
-
-    #[inline]
-    pub fn col_num(&self) -> usize {
-        self.parser.col_num()
-    }
-
-    #[inline]
-    pub fn update_expansion_lens(&mut self) {
-        let te_key = (clause_name!("term_expansion"), 2);
-        let ge_key = (clause_name!("goal_expansion"), 2);
-
-        let (tes_len, tes_q_len) = self.wam.code_repo.term_dir_entry_len(te_key);
-
-        self.term_expansion_lens.0 = tes_len;
-        self.term_expansion_lens.1 = tes_q_len;
-
-        let (ges_len, ges_q_len) = self.wam.code_repo.term_dir_entry_len(ge_key);
-
-        self.goal_expansion_lens.0 = ges_len;
-        self.goal_expansion_lens.1 = ges_q_len;
-    }
-
-    #[inline]
-    pub fn set_atom_tbl(&mut self, atom_tbl: TabledData<Atom>) {
-        self.parser.set_atom_tbl(atom_tbl);
-    }
-
-    #[inline]
-    pub fn eof(&mut self) -> Result<bool, ParserError> {
-	    self.parser.devour_whitespace()?; // eliminate dangling comments before checking for EOF.
-        Ok(self.stack.is_empty() && self.parser.eof()?)
-    }
-
-    pub fn rollback_expansion_code(&mut self) -> Result<ExpansionAdditionResult, ParserError> {
-        let te_len = self.term_expansion_lens.0;
-        let te_queue_len = self.term_expansion_lens.1;
-
-        let ge_len = self.goal_expansion_lens.0;
-        let ge_queue_len = self.goal_expansion_lens.1;
-
-        let term_expansion_additions = self.wam.code_repo.truncate_terms(
-            (clause_name!("term_expansion"), 2),
-            te_len,
-            te_queue_len,
-        );
-
-        let goal_expansion_additions = self.wam.code_repo.truncate_terms(
-            (clause_name!("goal_expansion"), 2),
-            ge_len,
-            ge_queue_len,
-        );
-
-        self.wam
-            .code_repo
-            .compile_hook(CompileTimeHook::TermExpansion)?;
-        self.wam
-            .code_repo
-            .compile_hook(CompileTimeHook::GoalExpansion)?;
-
-        Ok(ExpansionAdditionResult {
-            term_expansion_additions,
-            goal_expansion_additions,
-        })
-    }
-
-    fn enqueue_term(&mut self, term: Term) -> Result<(), ParserError> {
-        match term {
-            Term::Cons(_, head, tail) => {
-                let iter = extract_from_list(head, tail)?;
-                Ok(self.stack.extend(iter))
-            }
-            Term::Clause(..) | Term::Constant(_, Constant::Atom(..)) => {
-                Ok(self.stack.push(term))
-            }
-            _ => {
-                Err(ParserError::ExpectedTopLevelTerm)
-            }
-        }
-    }
-
-    fn parse_expansion_output(
-        &self,
-        term_string: &str,
-        op_dir: &OpDir,
-    ) -> Result<Term, ParserError> {
-        let mut stream = parsing_stream(term_string.trim().as_bytes())?;
-        let mut parser = Parser::new(&mut stream, self.parser.get_atom_tbl(), self.flags);
-
-        parser.read_term(composite_op!(
-            false,
-            &self.wam.indices.op_dir,
-            op_dir
-        ))
-    }
-
-    pub fn expand_term(&mut self, term: Term, op_dir: &OpDir) -> Result<Term, ParserError> {
-        let mut machine_st = MachineState::new();
-
-        self.stack.push(term);
-
-        while let Some(term) = self.stack.pop() {
-            match machine_st.try_expand_term(self.wam, &term, CompileTimeHook::TermExpansion) {
-                Some(term_string) => {
-                    let term = self.parse_expansion_output(term_string.as_str(), op_dir)?;
-                    self.enqueue_term(term)?;
-                }
-                None => {
-                    return Ok(term);
-                }
-            };
-        }
-
-        unreachable!()
-    }
-
-    pub fn read_term(&mut self, op_dir: &OpDir) -> Result<Term, ParserError> {
-        loop {
-            if let Some(term) = self.stack.pop() {
-                return Ok(self.expand_term(term, op_dir)?);
-            }
-
-            self.parser.reset();
-
-            let line_num = self.line_num();
-            let col_num = self.col_num();
-
-            let term = self.parser.read_term(composite_op!(
-                false,
-                &self.wam.indices.op_dir,
-                op_dir
-            ))?;
-
-            // preserve a copy of the original unexpanded term for warning scans,
-            // if that stage is reached.
-            self.top_level_terms.push((term.clone(), line_num, col_num));
-            self.stack.push(term);
-        }
-    }
-
-    pub(super)
-    fn expand_goals(
-        &mut self,
-        machine_st: &mut MachineState,
-        op_dir: &OpDir,
-        mut terms: VecDeque<Term>,
-    ) -> Result<Vec<Term>, ParserError> {
-        let mut results = vec![];
-
-        while let Some(term) = terms.pop_front() {
-            match machine_st.try_expand_term(self.wam, &term, CompileTimeHook::GoalExpansion) {
-                Some(term_string) => {
-                    let term = self.parse_expansion_output(term_string.as_str(), op_dir)?;
-
-                    match term {
-                        Term::Cons(_, head, tail) => {
-                            for term in extract_from_list(head, tail)? {
-                                terms.push_front(term);
-                            }
-                        }
-                        term => terms.push_front(term),
-                    };
-                }
-                None => results.push(term),
-            }
-        }
-
-        Ok(results)
-    }
-}
-
-impl MachineState {
-    pub(super)
-    fn print_with_locs(&self, addr: Addr, op_dir: &OpDir) -> PrinterOutputter {
-        let output = PrinterOutputter::new();
-        let mut printer = HCPrinter::from_heap_locs(&self, op_dir, output);
-        let mut max_var_length = 0;
-
-        for var in self.heap_locs.keys() {
-            max_var_length = std::cmp::max(var.len(), max_var_length);
-        }
-
-        printer.quoted = true;
-        printer.numbervars = true;
-
-        // the purpose of the offset is to avoid clashes with variable
-        // names that might occur after the addresses in the expanded
-        // term are substituted with the variable names in the
-        // pre-expansion term. This formula ensures that all generated
-        // "numbervars"- style variable names will be longer than the
-        // keys of the var_dict, and therefore not equal to any of
-        // them.
-        printer.numbervars_offset = Integer::from(10).pow(max_var_length as u32) * 26;
-        printer.print_strings_as_strs = true;
-        printer.drop_toplevel_spec();
-
-        printer.see_all_locs();
-
-        let mut output = printer.print(addr);
-
-        output.push_char('.');
-        output
-    }
-
-    // reset the machine, but keep the heap contents as they were.
-    // this prevents clashes between underscored variable names in the
-    // same query.
-    fn reset_with_heap_preservation(&mut self) {
-        let heap = self.heap.take();
-        self.reset();
-        self.heap = heap;
-    }
-
-    fn try_expand_term(
-        &mut self,
-        wam: &mut Machine,
-        term: &Term,
-        hook: CompileTimeHook,
-    ) -> Option<String> {
-        let term_write_result = write_term_to_heap(term, self);
-        let h = self.heap.h();
-
-        self[temp_v!(1)] = Addr::HeapCell(term_write_result.heap_loc);
-        self.heap.push(HeapCellValue::Addr(Addr::HeapCell(h)));
-        self[temp_v!(2)] = Addr::HeapCell(h);
-
-        let code = vec![call_clause!(ClauseType::Hook(hook), 2, 0, true)];
-        wam.code_repo.cached_query = code;
-
-        self.cp = LocalCodePtr::TopLevel(0, 0);
-
-        self.at_end_of_expansion = false;
-        self.flags.double_quotes = DoubleQuotes::Chars;
-
-        self.query_stepper(
-            &mut wam.indices,
-            &mut MachinePolicies::default(),
-            &mut wam.code_repo,
-            &mut readline::input_stream(),
-            &mut Stream::stdout(),
-        );
-
-        if self.fail || self.at_end_of_expansion {
-            self.reset_with_heap_preservation();
-            None
-        } else {
-            let TermWriteResult { var_dict, .. } = term_write_result;
-
-            self.heap_locs = var_dict;
-            let output = self.print_with_locs(Addr::HeapCell(h), &wam.indices.op_dir);
-
-            self.reset_with_heap_preservation();
-
-            Some(output.result())
-        }
-    }
-}

src/machine/term_stream.rs

@@ -0,0 +1,149 @@
+use crate::prolog_parser::ast::*;
+use crate::prolog_parser::parser::*;
+
+use crate::machine::*;
+use crate::machine::machine_errors::CompilationError;
+use crate::machine::preprocessor::*;
+
+use indexmap::IndexSet;
+
+use std::collections::VecDeque;
+use std::fmt;
+
+pub(crate) trait TermStream : Sized {
+    type Evacuable;
+
+    fn next(&mut self, op_dir: &CompositeOpDir) -> Result<Term, CompilationError>;
+    fn eof(&mut self) -> Result<bool, CompilationError>;
+    fn listing_src(&self) -> &ListingSource;
+    fn evacuate<'a>(loader: Loader<'a, Self>) -> Result<Self::Evacuable, SessionError>;
+}
+
+#[derive(Debug)]
+pub(super) struct BootstrappingTermStream<'a> {
+    listing_src: ListingSource,
+    parser: Parser<'a, Stream>,
+}
+
+impl<'a> BootstrappingTermStream<'a> {
+    #[inline]
+    pub(super)
+    fn from_prolog_stream(
+        stream: &'a mut PrologStream,
+        atom_tbl: TabledData<Atom>,
+        flags: MachineFlags,
+        listing_src: ListingSource,
+    ) -> Self {
+        let parser = Parser::new(stream, atom_tbl, flags);
+        Self { parser, listing_src }
+    }
+}
+
+impl<'a> TermStream for BootstrappingTermStream<'a> {
+    type Evacuable = CompilationTarget;
+
+    #[inline]
+    fn next(&mut self, op_dir: &CompositeOpDir) -> Result<Term, CompilationError> {
+        self.parser.reset();
+        self.parser.read_term(op_dir)
+            .map_err(CompilationError::from)
+    }
+
+    #[inline]
+    fn eof(&mut self) -> Result<bool, CompilationError> {
+	    self.parser.devour_whitespace()?; // eliminate dangling comments before checking for EOF.
+        Ok(self.parser.eof()?)
+    }
+
+    #[inline]
+    fn listing_src(&self) -> &ListingSource {
+        &self.listing_src
+    }
+
+    fn evacuate(mut loader: Loader<Self>) -> Result<Self::Evacuable, SessionError> {
+        if !loader.predicates.is_empty() {
+            loader.compile_and_submit()?;
+        }
+
+        loader.load_state.retraction_info.reset(
+            loader.load_state.wam.code_repo.code.len(),
+        );
+
+        loader.load_state.remove_module_op_exports();
+
+        Ok(loader.load_state.compilation_target.take())
+    }
+}
+
+pub struct LiveTermStream {
+    pub(super) term_queue: VecDeque<Term>,
+    pub(super) listing_src: ListingSource,
+}
+
+impl LiveTermStream {
+    #[inline]
+    pub(super)
+    fn new(listing_src: ListingSource) -> Self {
+        Self {
+            term_queue: VecDeque::new(),
+            listing_src,
+        }
+    }
+}
+
+pub struct LoadStatePayload {
+    pub(super) term_stream: LiveTermStream,
+    pub(super) compilation_target: CompilationTarget,
+    pub(super) retraction_info: RetractionInfo,
+    pub(super) module_op_exports: Vec<(OpDecl, Option<(usize, Specifier)>)>,
+    pub(super) non_counted_bt_preds: IndexSet<PredicateKey>,
+    pub(super) preprocessor: Preprocessor,
+    pub(super) predicates: Vec<PredicateClause>,
+    pub(super) clause_clauses: Vec<(Term, Term)>,
+}
+
+impl fmt::Debug for LoadStatePayload {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(fmt, "LoadStatePayload")
+    }
+}
+
+impl LoadStatePayload {
+    pub(super)
+    fn new(wam: &Machine) -> Self {
+        Self {
+            term_stream: LiveTermStream::new(ListingSource::User),
+            compilation_target: CompilationTarget::default(),
+            retraction_info: RetractionInfo::new(wam.code_repo.code.len()),
+            module_op_exports: vec![],
+            non_counted_bt_preds: IndexSet::new(),
+            preprocessor: Preprocessor::new(wam.machine_st.flags),
+            predicates: vec![],
+            clause_clauses: vec![],
+        }
+    }
+}
+
+impl TermStream for LiveTermStream {
+    type Evacuable = LoadStatePayload;
+
+    #[inline]
+    fn next(&mut self, _: &CompositeOpDir) -> Result<Term, CompilationError> {
+        Ok(self.term_queue.pop_front().unwrap())
+    }
+
+    #[inline]
+    fn eof(&mut self) -> Result<bool, CompilationError> {
+        return Ok(self.term_queue.is_empty());
+    }
+
+    #[inline]
+    fn listing_src(&self) -> &ListingSource {
+        &self.listing_src
+    }
+
+    #[inline]
+    fn evacuate(loader: Loader<Self>) -> Result<LoadStatePayload, SessionError> {
+        Ok(loader.to_load_state_payload())
+    }
+}