refactor to actual modules

src/prolog/machine/and_stack.rs

@@ -0,0 +1,91 @@
+use prolog::machine::machine_indices::*;
+
+use std::ops::{Index, IndexMut};
+use std::vec::Vec;
+
+#[derive(Clone)]
+pub struct Frame {
+    pub global_index: usize,
+    pub e: usize,
+    pub cp: LocalCodePtr,
+    pub special_form_cp: LocalCodePtr,
+    perms: Vec<Addr>
+}
+
+impl Frame {
+    fn new(global_index: usize, fr: usize, e: usize, cp: LocalCodePtr, n: usize) -> Self {
+        Frame {
+            global_index,
+            e: e,
+            cp: cp,
+            special_form_cp: LocalCodePtr::default(),
+            perms: (1 .. n+1).map(|i| Addr::StackCell(fr, i)).collect()
+        }
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.perms.len()
+    }
+}
+
+pub struct AndStack(Vec<Frame>);
+
+impl AndStack {
+    pub fn new() -> Self {
+        AndStack(Vec::new())
+    }
+
+    pub fn push(&mut self, global_index: usize, e: usize, cp: LocalCodePtr, n: usize) {
+        let len = self.0.len();
+        self.0.push(Frame::new(global_index, len, e, cp, n));
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    pub fn clear(&mut self) {
+        self.0.clear()
+    }
+
+    pub fn resize(&mut self, fr: usize, n: usize) {
+        let len = self[fr].perms.len();
+
+        if len < n {
+            self[fr].perms.reserve(n - len);
+
+            for i in len .. n {
+                self[fr].perms.push(Addr::StackCell(fr, i));
+            }
+        }
+    }
+}
+
+impl Index<usize> for AndStack {
+    type Output = Frame;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        self.0.index(index)
+    }
+}
+
+impl IndexMut<usize> for AndStack {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        self.0.index_mut(index)
+    }
+}
+
+impl Index<usize> for Frame {
+    type Output = Addr;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        self.perms.index(index - 1)
+    }
+}
+
+impl IndexMut<usize> for Frame {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        self.perms.index_mut(index - 1)
+    }
+}

src/prolog/machine/code_repo.rs

@@ -0,0 +1,124 @@
+use prolog_parser::ast::MachineFlags;
+
+use prolog::clause_types::*;
+use prolog::codegen::*;
+use prolog::debray_allocator::*;
+use prolog::forms::*;
+use prolog::instructions::*;
+use prolog::machine::compile::*;
+use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
+
+use std::collections::VecDeque;
+
+pub struct CodeRepo {
+    pub(super) cached_query: Code,
+    pub(super) goal_expanders: Code,
+    pub(super) term_expanders: Code,
+    pub(super) code: Code,
+    pub(super) in_situ_code: Code,
+    pub(super) term_dir: TermDir
+}
+
+impl CodeRepo {
+    #[inline]
+    pub(super) fn new() -> Self {
+        CodeRepo {
+            cached_query: vec![],
+            goal_expanders: Code::new(),
+            term_expanders: Code::new(),
+            code: Code::new(),
+            in_situ_code: Code::new(),
+            term_dir: TermDir::new()
+        }
+    }
+
+    #[inline]
+    pub fn term_dir_entry_len(&self, key: PredicateKey) -> (usize, usize) {
+        self.term_dir.get(&key)
+            .map(|entry| ((entry.0).0.len(), entry.1.len()))
+            .unwrap_or((0,0))
+    }
+
+    #[inline]
+    pub fn truncate_terms(&mut self, key: PredicateKey, len: usize, queue_len: usize)
+                          -> (Predicate, VecDeque<TopLevel>)
+    {
+        self.term_dir.get_mut(&key)
+            .map(|entry| (Predicate((entry.0).0.drain(len ..).collect()),
+                          entry.1.drain(queue_len ..).collect()))
+            .unwrap_or((Predicate::new(), VecDeque::from(vec![])))
+    }
+
+    pub fn add_in_situ_result(&mut self, result: &CompiledResult, in_situ_code_dir: &mut InSituCodeDir,
+                              flags: MachineFlags)
+                              -> Result<(), SessionError>
+    {
+        let (ref decl, ref queue) = result;
+        let (name, arity) = decl.0.first().and_then(|cl| {
+            let arity = cl.arity();
+            cl.name().map(|name| (name, arity))
+        }).ok_or(SessionError::NamelessEntry)?;
+
+        let p = self.in_situ_code.len();
+        in_situ_code_dir.insert((name, arity), p);
+
+        let mut cg = CodeGenerator::<DebrayAllocator>::new(true, flags);
+        // clone the decl to avoid the need to wipe its register cells later.
+        let mut decl_code = cg.compile_predicate(&decl.0.clone())?;
+
+        compile_appendix(&mut decl_code, queue, true, flags)?;
+
+        self.in_situ_code.extend(decl_code.into_iter());
+        Ok(())
+    }
+
+    #[inline]
+    pub(super)
+    fn size_of_cached_query(&self) -> usize {
+        self.cached_query.len()
+    }
+
+    pub(super)
+    fn lookup_instr<'a>(&'a self, last_call: bool, p: &CodePtr) -> Option<RefOrOwned<'a, Line>>
+    {
+        match p {
+            &CodePtr::Local(LocalCodePtr::UserGoalExpansion(p)) =>
+                if p < self.goal_expanders.len() {
+                    Some(RefOrOwned::Borrowed(&self.goal_expanders[p]))
+                } else {
+                    None
+                },
+            &CodePtr::Local(LocalCodePtr::UserTermExpansion(p)) =>
+                if p < self.term_expanders.len() {
+                    Some(RefOrOwned::Borrowed(&self.term_expanders[p]))
+                } else {
+                    None
+                },
+            &CodePtr::Local(LocalCodePtr::TopLevel(_, p)) =>
+                if p < self.cached_query.len() {
+                    Some(RefOrOwned::Borrowed(&self.cached_query[p]))
+                } else {
+                    None
+                },
+            &CodePtr::Local(LocalCodePtr::InSituDirEntry(p)) =>
+                Some(RefOrOwned::Borrowed(&self.in_situ_code[p])),
+            &CodePtr::Local(LocalCodePtr::DirEntry(p)) =>
+                Some(RefOrOwned::Borrowed(&self.code[p])),
+            &CodePtr::BuiltInClause(ref built_in, _) => {
+                let call_clause = call_clause!(ClauseType::BuiltIn(built_in.clone()),
+                                               built_in.arity(),
+                                               0, last_call);
+                Some(RefOrOwned::Owned(call_clause))
+            },
+            &CodePtr::CallN(arity, _) => {
+                let call_clause = call_clause!(ClauseType::CallN, arity, 0, last_call);
+                Some(RefOrOwned::Owned(call_clause))
+            },
+            &CodePtr::VerifyAttrInterrupt(p) =>
+                Some(RefOrOwned::Borrowed(&self.code[p])),
+            &CodePtr::DynamicTransaction(..) =>
+                None
+        }
+    }
+}

src/prolog/machine/compile.rs

@@ -0,0 +1,576 @@
+use prolog_parser::ast::*;
+use prolog_parser::tabled_rc::TabledData;
+
+use prolog::instructions::*;
+use prolog::codegen::*;
+use prolog::debray_allocator::*;
+use prolog::forms::*;
+use prolog::machine::*;
+use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
+use prolog::machine::term_expansion::{ExpansionAdditionResult};
+use prolog::machine::toplevel::*;
+
+use std::cell::Cell;
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::io::Read;
+use std::mem;
+
+#[allow(dead_code)]
+fn print_code(code: &Code) {
+    for clause in code {
+        match clause {
+            &Line::Arithmetic(ref arith) =>
+                println!("{}", arith),
+            &Line::Fact(ref fact_instr) =>
+                println!("{}", fact_instr),
+            &Line::Cut(ref cut) =>
+                println!("{}", cut),
+            &Line::Choice(ref choice) =>
+                println!("{}", choice),
+            &Line::Control(ref control) =>
+                println!("{}", control),
+            &Line::IndexedChoice(ref choice) =>
+                println!("{}", choice),
+            &Line::Indexing(ref indexing) =>
+                println!("{}", indexing),
+            &Line::Query(ref query_instr) =>
+                println!("{}", query_instr)
+        }
+    }
+}
+
+pub type PredicateCompileQueue = (Predicate, VecDeque<TopLevel>);
+
+// throw errors if declaration or query found.
+fn compile_relation(tl: &TopLevel, non_counted_bt: bool, flags: MachineFlags)
+                    -> Result<Code, ParserError>
+{
+    let mut cg = CodeGenerator::<DebrayAllocator>::new(non_counted_bt, flags);
+
+    match tl {
+        &TopLevel::Declaration(_) | &TopLevel::Query(_) =>
+            Err(ParserError::ExpectedRel),
+        &TopLevel::Predicate(ref clauses) =>
+            cg.compile_predicate(&clauses.0),
+        &TopLevel::Fact(ref fact) =>
+            Ok(cg.compile_fact(fact)),
+        &TopLevel::Rule(ref rule) =>
+            cg.compile_rule(rule)
+    }
+}
+
+// set first jmp_by_call or jmp_by_index instruction to code.len() -
+// idx, where idx is the place it occurs. It only does this to the
+// *first* uninitialized jmp index it encounters, then returns.
+fn set_first_index(code: &mut Code)
+{
+    let code_len = code.len();
+
+    for (idx, line) in code.iter_mut().enumerate() {
+        match line {
+            &mut Line::Control(ControlInstruction::JmpBy(_, ref mut offset, ..)) if *offset == 0 => {
+                *offset = code_len - idx;
+                break;
+            },
+            _ => {}
+        };
+    }
+}
+
+pub fn compile_appendix(code: &mut Code, queue: &VecDeque<TopLevel>, non_counted_bt: bool,
+                        flags: MachineFlags)
+                        -> Result<(), ParserError>
+{
+    for tl in queue.iter() {
+        set_first_index(code);
+        code.append(&mut compile_relation(tl, non_counted_bt, flags)?);
+    }
+
+    Ok(())
+}
+
+impl CodeRepo {
+    pub fn compile_hook(&mut self, hook: CompileTimeHook, flags: MachineFlags)
+                        -> Result<(), ParserError>
+    {
+        let key = (hook.name(), hook.arity());
+        match self.term_dir.get(&key) {
+            Some(preds) => {
+                let mut cg = CodeGenerator::<DebrayAllocator>::new(false, flags);
+                let mut code = cg.compile_predicate(&(preds.0).0)?;
+
+                compile_appendix(&mut code, &preds.1, false, flags)?;
+
+                Ok(match hook {
+                    CompileTimeHook::UserTermExpansion | CompileTimeHook::TermExpansion =>
+                        self.term_expanders = code,
+                    CompileTimeHook::UserGoalExpansion | CompileTimeHook::GoalExpansion =>
+                        self.goal_expanders = code
+                })
+            },
+            None => Ok(())
+        }
+    }
+}
+
+fn compile_query(terms: Vec<QueryTerm>, queue: VecDeque<TopLevel>, flags: MachineFlags)
+                 -> Result<(Code, AllocVarDict), ParserError>
+{
+    // count backtracking inferences.
+    let mut cg = CodeGenerator::<DebrayAllocator>::new(false, flags);
+    let mut code = try!(cg.compile_query(&terms));
+
+    compile_appendix(&mut code, &queue, false, flags)?;
+    Ok((code, cg.take_vars()))
+}
+
+fn compile_decl(wam: &mut Machine, compiler: &mut ListingCompiler, decl: Declaration)
+                -> Result<IndexStore, SessionError>
+{
+    let flags = wam.machine_flags();
+    let mut indices = default_index_store!(wam.indices.atom_tbl.clone());
+    let wam_indices = &mut wam.indices;
+
+    compiler.process_decl(decl, &mut wam.code_repo, wam_indices, &mut indices, flags)?;
+
+    Ok(indices)
+}
+
+pub fn compile_term(wam: &mut Machine, packet: TopLevelPacket) -> EvalSession
+{
+    match packet {
+        TopLevelPacket::Query(terms, queue) =>
+            match compile_query(terms, queue, wam.machine_flags()) {
+                Ok((mut code, vars)) => wam.submit_query(code, vars),
+                Err(e) => EvalSession::from(e)
+            },
+        TopLevelPacket::Decl(TopLevel::Declaration(decl), _) => {
+            let mut compiler = ListingCompiler::new(&wam.code_repo);
+            let indices = try_eval_session!(compile_decl(wam, &mut compiler, decl));
+
+            try_eval_session!(compiler.add_code(wam, vec![], indices));
+
+            EvalSession::EntrySuccess
+        },
+        _ => EvalSession::from(SessionError::UserPrompt)
+    }
+}
+
+pub struct GatherResult {
+    dynamic_clause_map: DynamicClauseMap,
+    pub(crate) worker_results: Vec<PredicateCompileQueue>,
+    toplevel_results: Vec<PredicateCompileQueue>,
+    toplevel_indices: IndexStore,
+    addition_results: ExpansionAdditionResult
+}
+
+pub struct ListingCompiler {
+    non_counted_bt_preds: HashSet<PredicateKey>,
+    module: Option<Module>,
+    user_term_dir: TermDir,
+    orig_term_expansion_lens: (usize, usize),
+    orig_goal_expansion_lens: (usize, usize)
+}
+
+impl ListingCompiler {
+    #[inline]
+    pub fn new(code_repo: &CodeRepo) -> Self {
+        ListingCompiler {
+            non_counted_bt_preds: HashSet::new(),
+            module: None,
+            user_term_dir: TermDir::new(),
+            orig_term_expansion_lens: code_repo.term_dir_entry_len((clause_name!("term_expansion"), 2)),
+            orig_goal_expansion_lens: code_repo.term_dir_entry_len((clause_name!("goal_expansion"), 2)),
+        }
+    }
+
+    /*
+    Replace calls to self with a localized index cell, not available to the global CodeIndex.
+    This is done to implement logical update semantics for dynamic database updates.
+     */
+    fn localize_self_calls(&mut self, name: ClauseName, arity: usize, code: &mut Code, p: usize)
+    {
+        let self_idx = CodeIndex::default();
+        set_code_index!(self_idx, IndexPtr::Index(p), self.get_module_name());
+
+        for instr in code.iter_mut() {
+            if let &mut Line::Control(ControlInstruction::CallClause(ref mut ct, ..)) = instr {
+                match ct {
+                    &mut ClauseType::Named(ref ct_name, ct_arity, ref mut idx)
+                        if ct_name == &name && arity == ct_arity => {
+                            *idx = self_idx.clone();
+                        },
+                    &mut ClauseType::Op(ref op_decl, ref mut idx)
+                        if op_decl.name() == name && op_decl.arity() == arity => {
+                            *idx = self_idx.clone();
+                        },
+                    _ => {}
+                }
+            }
+        }
+    }
+
+    fn use_module(&mut self, submodule: ClauseName, code_repo: &mut CodeRepo,
+                  flags: MachineFlags, wam_indices: &mut IndexStore,
+                  indices: &mut IndexStore)
+                  -> Result<(), SessionError>
+    {
+        let mod_name = self.get_module_name();
+
+        if let Some(mut submodule) = wam_indices.take_module(submodule) {
+            indices.use_module(code_repo, flags, &submodule)?;
+
+            if let &mut Some(ref mut module) = &mut self.module {
+                module.remove_module(mod_name, &submodule);
+                module.use_module(code_repo, flags, &submodule)?;
+            } else {
+                submodule.inserted_expansions = true;
+                wam_indices.remove_module(clause_name!("user"), &submodule);
+            }
+
+            Ok(wam_indices.insert_module(submodule))
+        } else {
+            Err(SessionError::ModuleNotFound)
+        }
+    }
+
+    fn use_qualified_module(&mut self, submodule: ClauseName, code_repo: &mut CodeRepo,
+                            flags: MachineFlags, exports: &Vec<PredicateKey>,
+                            wam_indices: &mut IndexStore, indices: &mut IndexStore)
+                            -> Result<(), SessionError>
+    {
+        let mod_name = self.get_module_name();
+
+        if let Some(mut submodule) = wam_indices.take_module(submodule) {
+            indices.use_qualified_module(code_repo, flags, &submodule, exports)?;
+
+            if let &mut Some(ref mut module) = &mut self.module {
+                module.remove_module(mod_name, &submodule);
+                module.use_qualified_module(code_repo, flags, &submodule, exports)?;
+            } else {
+                submodule.inserted_expansions = true;
+                wam_indices.remove_module(clause_name!("user"), &submodule);
+            }
+
+            Ok(wam_indices.insert_module(submodule))
+        } else {
+            Err(SessionError::ModuleNotFound)
+        }
+    }
+
+    #[inline]
+    fn get_module_name(&self) -> ClauseName {
+        self.module.as_ref()
+            .map(|module| module.module_decl.name.clone())
+            .unwrap_or(ClauseName::BuiltIn("user"))
+    }
+
+    fn add_clause_code(&mut self, dynamic_clause_map: DynamicClauseMap, wam: &mut Machine)
+                       -> Result<(), SessionError>
+    {
+        let mut code = vec![];
+        let mut pi_to_loc = HashMap::new();
+
+        for ((name, arity), heads_and_tails) in dynamic_clause_map {
+            if heads_and_tails.is_empty() {
+                continue;
+            }
+
+            let predicate = Predicate(heads_and_tails.into_iter().map(|(head, tail)| {
+                let clause = Term::Clause(Cell::default(), clause_name!("clause"),
+                                          vec![Box::new(head), Box::new(tail)],
+                                          None);
+                PredicateClause::Fact(clause)
+            }).collect());
+
+            let p = code.len() + wam.code_size();
+            let mut decl_code = compile_relation(&TopLevel::Predicate(predicate), false,
+                                                 wam.machine_flags())?;
+
+            compile_appendix(&mut decl_code, &VecDeque::new(), false, wam.machine_flags())?;
+            
+            pi_to_loc.insert((name, arity), p);
+            code.extend(decl_code.into_iter());
+        }
+
+        wam.code_repo.code.extend(code.into_iter());
+
+        for ((name, arity), p) in pi_to_loc {
+            let entry = wam.indices.dynamic_code_dir.entry((name, arity))
+                           .or_insert(DynamicPredicateInfo::default());
+            entry.clauses_subsection_p = p;
+        }
+
+        Ok(())
+    }
+
+    pub(crate)
+    fn generate_code(&mut self, decls: Vec<PredicateCompileQueue>, wam: &Machine,
+                     code_dir: &mut CodeDir)
+                     -> Result<Code, SessionError>
+    {
+        let mut code = vec![];
+
+        for (decl, queue) in decls {
+            let (name, arity) = decl.predicate_indicator().ok_or(SessionError::NamelessEntry)?;
+            let non_counted_bt = self.non_counted_bt_preds.contains(&(name.clone(), arity));
+
+            let p = code.len() + wam.code_size();
+            let mut decl_code = compile_relation(&TopLevel::Predicate(decl), non_counted_bt,
+                                                 wam.machine_flags())?;
+
+            compile_appendix(&mut decl_code, &queue, non_counted_bt, wam.machine_flags())?;
+
+            let idx = code_dir.entry((name.clone(), arity)).or_insert(CodeIndex::default());
+            set_code_index!(idx, IndexPtr::Index(p), self.get_module_name());
+
+            self.localize_self_calls(name, arity, &mut decl_code, p);
+            code.extend(decl_code.into_iter());
+        }
+
+        Ok(code)
+    }
+
+    fn add_code(&mut self, wam: &mut Machine, code: Code, mut indices: IndexStore)
+                -> Result<(), SessionError>
+    {
+        let code_dir = mem::replace(&mut indices.code_dir, CodeDir::new());
+        let op_dir   = mem::replace(&mut indices.op_dir, OpDir::new());
+
+        if let Some(mut module) = self.module.take() {
+            module.code_dir.extend(as_module_code_dir(code_dir));
+            module.op_dir.extend(op_dir.into_iter());
+
+            wam.add_module(module, code);
+        } else {
+            wam.add_batched_code(code, code_dir)?;
+            wam.add_batched_ops(op_dir);
+        }
+
+        Ok(())
+    }
+
+    fn add_non_counted_bt_flag(&mut self, name: ClauseName, arity: usize) {
+        self.non_counted_bt_preds.insert((name, arity));
+    }
+
+    fn add_term_dir_terms(&mut self, hook: CompileTimeHook, code_repo: &mut CodeRepo,
+                          key: PredicateKey, clause: PredicateClause, queue: VecDeque<TopLevel>)
+                          -> (usize, usize)
+    {
+        let preds = code_repo.term_dir.entry(key.clone())
+            .or_insert((Predicate::new(), VecDeque::from(vec![])));
+
+        let (mut len, mut queue_len) = ((preds.0).0.len(), preds.1.len());
+
+        if self.module.is_some() && hook.has_module_scope() {
+            let module_preds = self.user_term_dir.entry(key.clone())
+                .or_insert((Predicate::new(), VecDeque::from(vec![])));
+
+            (module_preds.0).0.push(clause);
+            module_preds.1.extend(queue.into_iter());
+
+            (preds.0).0.extend((module_preds.0).0.iter().cloned());
+            preds.1.extend(module_preds.1.iter().cloned());
+        } else {
+            let module_preds = self.user_term_dir.entry(key.clone())
+                .or_insert((Predicate::new(), VecDeque::from(vec![])));
+
+            len += 1;
+            queue_len += queue_len;
+
+            (preds.0).0.push(clause);
+            preds.1.extend(queue.into_iter());
+
+            (preds.0).0.extend((module_preds.0).0.iter().cloned());
+            preds.1.extend(module_preds.1.iter().cloned());
+        }
+
+        (len, queue_len)
+    }
+
+    fn process_decl(&mut self, decl: Declaration, code_repo: &mut CodeRepo,
+                    wam_indices: &mut IndexStore, indices: &mut IndexStore,
+                    flags: MachineFlags)
+                    -> Result<(), SessionError>
+    {
+        match decl {
+            Declaration::Hook(hook, clause, queue) => {
+                let key = (hook.name(), hook.arity());
+                let (len, queue_len) = self.add_term_dir_terms(hook, code_repo, key.clone(),
+                                                               clause, queue);
+
+                let result = code_repo.compile_hook(hook, flags).map_err(SessionError::from);
+                code_repo.truncate_terms(key, len, queue_len);
+
+                result
+            },
+            Declaration::NonCountedBacktracking(name, arity) =>
+                Ok(self.add_non_counted_bt_flag(name, arity)),
+            Declaration::Op(op_decl) =>
+                op_decl.submit(self.get_module_name(), &mut indices.op_dir),
+            Declaration::UseModule(name) =>
+                self.use_module(name, code_repo, flags, wam_indices, indices),
+            Declaration::UseQualifiedModule(name, exports) =>
+                self.use_qualified_module(name, code_repo, flags, &exports, wam_indices, indices),
+            Declaration::Module(module_decl) =>
+                if self.module.is_none() {
+                    let module_name = module_decl.name.clone();
+                    let atom_tbl = TabledData::new(module_name.to_rc());
+
+                    Ok(self.module = Some(Module::new(module_decl, atom_tbl)))
+                } else {
+                    Err(SessionError::from(ParserError::InvalidModuleDecl))
+                },
+            Declaration::Dynamic(..) => Ok(())
+        }
+    }
+
+    fn process_and_commit_decl<'a, R: Read>(&mut self, decl: Declaration,
+                                            worker: &mut TopLevelBatchWorker<'a, R>,
+                                            indices: &mut IndexStore, flags: MachineFlags)
+                                            -> Result<(), SessionError>
+    {
+        match &decl {
+            &Declaration::Dynamic(ref name, arity) => {
+                worker.dynamic_clause_map.entry((name.clone(), arity)).or_insert(vec![]);
+            },
+            &Declaration::Hook(hook, _, ref queue) if self.module.is_none() =>
+                worker.term_stream.incr_expansion_lens(hook.user_scope(), 1, queue.len()),
+            &Declaration::Hook(hook, _, ref queue) if !hook.has_module_scope() =>
+                worker.term_stream.incr_expansion_lens(hook, 1, queue.len()),
+            _ => {}
+        };
+
+        self.process_decl(decl, &mut worker.term_stream.code_repo,
+                          &mut worker.term_stream.indices, indices, flags)
+    }
+
+    pub(crate)
+    fn gather_items<R: Read>(&mut self, wam: &mut Machine, src: R, indices: &mut IndexStore)
+                             -> Result<GatherResult, SessionError>
+    {
+        let flags      = wam.machine_flags();
+        let atom_tbl   = wam.indices.atom_tbl.clone();
+        let mut worker = TopLevelBatchWorker::new(src, atom_tbl.clone(), flags,
+                                                  &mut wam.indices, &mut wam.policies,
+                                                  &mut wam.code_repo);
+
+        let mut toplevel_results = vec![];
+        let mut toplevel_indices = default_index_store!(atom_tbl.clone());
+
+        while let Some(decl) = worker.consume(indices)? {
+            if decl.is_module_decl() {
+                toplevel_indices.copy_and_swap(indices);
+                mem::swap(&mut worker.results, &mut toplevel_results);
+                worker.in_module = true;
+
+                self.process_and_commit_decl(decl, &mut worker, indices, flags)?;
+
+                if let &Some(ref module) = &self.module {
+                    worker.term_stream.set_atom_tbl(module.atom_tbl.clone());
+                }
+            } else {
+                self.process_and_commit_decl(decl, &mut worker, indices, flags)?;
+            }
+        }
+
+        let addition_results = worker.term_stream.rollback_expansion_code()?;
+
+        Ok(GatherResult {
+            worker_results: worker.results,
+            dynamic_clause_map: worker.dynamic_clause_map,
+            toplevel_results,
+            toplevel_indices,
+            addition_results
+        })
+    }
+
+    fn drop_expansions(&self, flags: MachineFlags, code_repo: &mut CodeRepo)
+    {
+        let (te_len, te_queue_len) = self.orig_term_expansion_lens;
+        let (ge_len, ge_queue_len) = self.orig_goal_expansion_lens;
+
+        code_repo.truncate_terms((clause_name!("term_expansion"), 2), te_len, te_queue_len);
+        code_repo.truncate_terms((clause_name!("goal_expansion"), 2), ge_len, ge_queue_len);
+
+        discard_result!(code_repo.compile_hook(CompileTimeHook::UserGoalExpansion, flags));
+        discard_result!(code_repo.compile_hook(CompileTimeHook::UserTermExpansion, flags));
+    }
+}
+
+fn compile_work<R: Read>(compiler: &mut ListingCompiler, wam: &mut Machine, src: R,
+                         mut indices: IndexStore)
+                         -> EvalSession
+{
+    let mut results = try_eval_session!(compiler.gather_items(wam, src, &mut indices));
+
+    let module_code = try_eval_session!(compiler.generate_code(results.worker_results, wam,
+                                                               &mut indices.code_dir));
+    let toplvl_code = try_eval_session!(compiler.generate_code(results.toplevel_results, wam,
+                                                               &mut results.toplevel_indices.code_dir));
+
+    if let Some(ref mut module) = &mut compiler.module {
+        module.term_expansions = results.addition_results.take_term_expansions();
+        module.goal_expansions = results.addition_results.take_goal_expansions();
+    }
+
+    let flags = wam.machine_flags();
+
+    try_eval_session!(wam.code_repo.compile_hook(CompileTimeHook::UserTermExpansion, flags));
+    try_eval_session!(wam.code_repo.compile_hook(CompileTimeHook::UserGoalExpansion, flags));
+
+    try_eval_session!(compiler.add_code(wam, module_code, indices));
+    try_eval_session!(compiler.add_code(wam, toplvl_code, results.toplevel_indices));
+
+    try_eval_session!(compiler.add_clause_code(results.dynamic_clause_map, wam));    
+    
+    EvalSession::EntrySuccess
+}
+
+/* This is a truncated version of compile_user_module, used for
+   compiling code composing special forms, ie. the code that calls
+   M:verify_attributes on attributed variables. */
+pub fn compile_special_form<R: Read>(wam: &mut Machine, src: R) -> Result<Code, SessionError>
+{
+    let mut indices = default_index_store!(wam.indices.atom_tbl.clone());
+    setup_indices(wam, &mut indices)?;
+
+    let mut compiler = ListingCompiler::new(&wam.code_repo);
+    let results = compiler.gather_items(wam, src, &mut indices)?;
+
+    compiler.generate_code(results.worker_results, wam, &mut indices.code_dir)
+}
+
+#[inline]
+pub fn compile_listing<R: Read>(wam: &mut Machine, src: R, indices: IndexStore) -> EvalSession
+{
+    let mut compiler = ListingCompiler::new(&wam.code_repo);
+
+    match compile_work(&mut compiler, wam, src, indices) {
+        EvalSession::Error(e) => {
+            compiler.drop_expansions(wam.machine_flags(), &mut wam.code_repo);
+            EvalSession::Error(e)
+        },
+        result => result
+    }
+}
+
+fn setup_indices(wam: &mut Machine, indices: &mut IndexStore) -> Result<(), SessionError> {
+    if let Some(builtins) = wam.indices.take_module(clause_name!("builtins")) {
+        let flags  = wam.machine_flags();
+        let result = indices.use_module(&mut wam.code_repo, flags, &builtins);
+
+        wam.indices.insert_module(builtins);
+        result
+    } else {
+        Err(SessionError::ModuleNotFound)
+    }
+}
+
+pub fn compile_user_module<R: Read>(wam: &mut Machine, src: R) -> EvalSession {
+    let mut indices = default_index_store!(wam.indices.atom_tbl.clone());
+    try_eval_session!(setup_indices(wam, &mut indices));
+    compile_listing(wam, src, indices)
+}

src/prolog/machine/copier.rs

@@ -0,0 +1,206 @@
+use prolog::machine::and_stack::*;
+use prolog::machine::machine_indices::*;
+
+use std::ops::IndexMut;
+
+type Trail = Vec<(Ref, HeapCellValue)>;
+
+pub(crate) trait CopierTarget: IndexMut<usize, Output=HeapCellValue>
+{
+    fn threshold(&self) -> usize;
+    fn push(&mut self, HeapCellValue);
+    fn store(&self, Addr) -> Addr;
+    fn deref(&self, Addr) -> Addr;
+    fn stack(&mut self) -> &mut AndStack;
+}
+
+pub(crate)
+fn copy_term<T: CopierTarget>(target: T, addr: Addr)
+{
+    let mut copy_term_state = CopyTermState::new(target);
+    copy_term_state.copy_term_impl(addr);
+}
+
+struct CopyTermState<T: CopierTarget> {
+    trail: Trail,
+    scan: usize,
+    old_h: usize,
+    target: T
+}
+
+impl<T: CopierTarget> CopyTermState<T> {
+    fn new(target: T) -> Self {
+        CopyTermState {
+            trail: vec![],
+            scan:  0,
+            old_h: target.threshold(),
+            target
+        }
+    }
+
+    #[inline]
+    fn value_at_scan(&mut self) -> &mut HeapCellValue {
+        let scan = self.scan;
+        &mut self.target[scan]
+    }
+
+    fn reinstantiate_var(&mut self, addr: Addr, threshold: usize)
+    {
+        match addr {
+            Addr::HeapCell(h) => {
+                self.target[threshold] = HeapCellValue::Addr(Addr::HeapCell(threshold));
+                self.target[h] = HeapCellValue::Addr(Addr::HeapCell(threshold));
+                self.trail.push((Ref::HeapCell(h), HeapCellValue::Addr(Addr::HeapCell(h))));
+            },
+            Addr::StackCell(fr, sc) => {
+                self.target[threshold] = HeapCellValue::Addr(Addr::HeapCell(threshold));
+                self.target.stack()[fr][sc] = Addr::HeapCell(threshold);
+                self.trail.push((Ref::StackCell(fr, sc), HeapCellValue::Addr(Addr::StackCell(fr, sc))));
+            },
+            Addr::AttrVar(h) => {
+                self.target[threshold] = HeapCellValue::Addr(Addr::AttrVar(threshold));
+                self.target[h] = HeapCellValue::Addr(Addr::AttrVar(threshold));
+                self.trail.push((Ref::AttrVar(h), HeapCellValue::Addr(Addr::AttrVar(h))));
+            },
+            _ => {}
+        }
+    }
+
+    fn copied_list(&mut self, addr: usize) -> bool {
+        if let HeapCellValue::Addr(Addr::Lis(addr)) = self.target[addr].clone() {
+            if addr >= self.old_h {
+                *self.value_at_scan() = HeapCellValue::Addr(Addr::Lis(addr));
+                self.scan += 1;
+                return true;
+            }
+        }
+
+        false
+    }
+
+    fn copy_list(&mut self, addr: usize) {
+        if self.copied_list(addr) {
+            return;
+        }
+
+        let threshold = self.target.threshold();
+        *self.value_at_scan() = HeapCellValue::Addr(Addr::Lis(threshold));
+
+        let hcv = self.target[addr].clone();
+        self.target.push(hcv.clone());
+
+        let ra = hcv.as_addr(threshold);
+        let rd = self.target.store(self.target.deref(ra));
+
+        match rd.clone() {
+            Addr::AttrVar(h) | Addr::HeapCell(h) if h >= self.old_h =>
+                self.target[threshold] = HeapCellValue::Addr(rd),
+            ra @ Addr::AttrVar(_) | ra @ Addr::HeapCell(..) | ra @ Addr::StackCell(..) =>
+                if ra == rd {
+                    self.reinstantiate_var(ra, threshold);
+                } else {
+                    self.target[threshold] = HeapCellValue::Addr(ra);
+                },
+            _ => {
+                self.trail.push((Ref::HeapCell(addr), self.target[addr].clone()));
+                self.target[addr] = HeapCellValue::Addr(Addr::Lis(threshold))
+            }
+        };
+
+        let hcv = self.target[addr + 1].clone();
+        self.target.push(hcv);
+
+        self.scan += 1;
+    }
+
+    fn copy_var(&mut self, addr: Addr) {
+        let rd = self.target.store(self.target.deref(addr.clone()));
+
+        match rd.clone() {
+            Addr::AttrVar(h) | Addr::HeapCell(h) if h >= self.old_h => {
+                *self.value_at_scan() = HeapCellValue::Addr(rd);
+                self.scan += 1;
+            },
+            Addr::AttrVar(h) if addr == rd => {
+                let threshold = self.target.threshold();
+                self.target.push(HeapCellValue::Addr(Addr::AttrVar(threshold)));
+
+                let list_val = self.target[h + 1].clone();
+                self.target.push(list_val);
+
+                self.reinstantiate_var(addr, threshold);
+                *self.value_at_scan() = HeapCellValue::Addr(Addr::AttrVar(threshold));
+            },
+            _ if addr == rd => {
+                let scan = self.scan;
+                self.reinstantiate_var(addr, scan);
+                self.scan += 1;
+            },
+            _ => *self.value_at_scan() = HeapCellValue::Addr(rd)
+        }
+    }
+
+    fn copy_structure(&mut self, addr: usize) {
+        match self.target[addr].clone() {
+            HeapCellValue::NamedStr(arity, name, fixity) => {
+                let threshold = self.target.threshold();
+
+                *self.value_at_scan() = HeapCellValue::Addr(Addr::Str(threshold));
+                self.target[addr] = HeapCellValue::Addr(Addr::Str(threshold));
+
+                self.trail.push((Ref::HeapCell(addr),
+                                 HeapCellValue::NamedStr(arity, name.clone(), fixity)));
+
+                self.target.push(HeapCellValue::NamedStr(arity, name, fixity));
+
+                for i in 0 .. arity {
+                    let hcv = self.target[addr + 1 + i].clone();
+                    self.target.push(hcv);
+                }
+            },
+            HeapCellValue::Addr(Addr::Str(addr)) =>
+                *self.value_at_scan() = HeapCellValue::Addr(Addr::Str(addr)),
+            _ => {}
+        }
+
+        self.scan += 1;
+    }
+
+    fn copy_term_impl(&mut self, addr: Addr) {
+        self.scan = self.target.threshold();
+        self.target.push(HeapCellValue::Addr(addr));
+
+        while self.scan < self.target.threshold() {
+            match self.value_at_scan().clone() {
+                HeapCellValue::NamedStr(..) =>
+                    self.scan += 1,
+                HeapCellValue::Addr(addr) =>
+                    match addr {
+                        Addr::Lis(addr) =>
+                            self.copy_list(addr),
+                        addr @ Addr::AttrVar(_)
+                      | addr @ Addr::HeapCell(_)
+                      | addr @ Addr::StackCell(..) =>
+                            self.copy_var(addr),
+                        Addr::Str(addr) =>
+                            self.copy_structure(addr),
+                        Addr::Con(_) =>
+                            self.scan += 1
+                    }
+            }
+        }
+
+        self.unwind_trail();
+    }
+
+    fn unwind_trail(&mut self) {
+        for (r, value) in self.trail.drain(0 ..) {
+            match r {
+                Ref::AttrVar(h) | Ref::HeapCell(h) =>
+                    self.target[h] = value,
+                Ref::StackCell(fr, sc) =>
+                    self.target.stack()[fr][sc] = value.as_addr(0)
+            }
+        }
+    }
+}

src/prolog/machine/dynamic_database.rs

@@ -1,9 +1,8 @@
 use prolog_parser::ast::*;
 
-use prolog::compile::*;
 use prolog::heap_print::*;
-use prolog::instructions::*;
 use prolog::machine::*;
+use prolog::machine::compile::*;
 use prolog::machine::machine_errors::*;
 use prolog::num::ToPrimitive;
 

src/prolog/machine/heap.rs

@@ -0,0 +1,79 @@
+use prolog_parser::ast::*;
+
+use prolog::machine::machine_indices::*;
+
+use std::ops::{Index, IndexMut};
+
+pub struct Heap {
+    heap: Vec<HeapCellValue>,
+    pub h: usize,
+}
+
+impl Heap {
+    pub fn with_capacity(cap: usize) -> Self {
+        Heap { heap: Vec::with_capacity(cap),
+               h: 0 }
+    }
+
+    #[inline]
+    pub fn push(&mut self, val: HeapCellValue) {
+        self.heap.push(val);
+        self.h += 1;
+    }
+
+    #[inline]
+    pub fn truncate(&mut self, h: usize) {
+        self.h = h;
+        self.heap.truncate(h);
+    }
+
+    #[inline]
+    pub fn last(&self) -> Option<&HeapCellValue> {
+        self.heap.last()
+    }
+
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.heap.len()
+    }
+
+    pub fn append(&mut self, vals: Vec<HeapCellValue>) {
+        let n = vals.len();
+
+        self.heap.extend(vals.into_iter());
+        self.h += n;
+    }
+
+    pub fn clear(&mut self) {
+        self.heap.clear();
+        self.h = 0;
+    }
+
+    pub fn to_list<Iter: Iterator<Item=Addr>>(&mut self, values: Iter) -> usize {
+        let head_addr = self.h;
+
+        for value in values {
+            let h = self.h;
+
+            self.push(HeapCellValue::Addr(Addr::Lis(h+1)));
+            self.push(HeapCellValue::Addr(value));
+        }
+
+        self.push(HeapCellValue::Addr(Addr::Con(Constant::EmptyList)));
+        head_addr
+    }
+}
+
+impl Index<usize> for Heap {
+    type Output = HeapCellValue;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.heap[index]
+    }
+}
+
+impl IndexMut<usize> for Heap {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        &mut self.heap[index]
+    }
+}

src/prolog/machine/machine_errors.rs

@@ -1,6 +1,6 @@
 use prolog_parser::ast::*;
 
-use prolog::instructions::*;
+use prolog::machine::machine_indices::*;
 use prolog::machine::machine_state::*;
 use prolog::num::bigint::BigInt;
 
@@ -81,26 +81,26 @@ impl MachineError {
 
     pub(super) fn permission_error(err: PermissionError, pred_str: ClauseName) -> Self {
         let pred_str = HeapCellValue::Addr(Addr::Con(Constant::Atom(pred_str, None)));
-        
+
         let err = vec![heap_atom!(err.as_str()), pred_str];
         let mut stub = functor!("permission_error", 2);
-        
+
         stub.extend(err.into_iter());
 
         MachineError { stub, from: ErrorProvenance::Constructed }
     }
-    
+
     pub(super) fn syntax_error(h: usize, err: ParserError) -> Self {
-        let err = vec![heap_atom!(err.as_str())];        
+        let err = vec![heap_atom!(err.as_str())];
 
         let mut stub = if err.len() == 1 {
-            functor!("syntax_error", 1)            
+            functor!("syntax_error", 1)
         } else {
             functor!("syntax_error", 1, [heap_str!(h + 2)])
         };
-        
+
         stub.extend(err.into_iter());
-        
+
         MachineError { stub, from: ErrorProvenance::Constructed }
     }
 
@@ -153,7 +153,7 @@ impl PermissionError {
         }
     }
 }
-    
+
 // from 7.12.2 b) of 13211-1:1995
 #[derive(Clone, Copy)]
 pub enum ValidType {
@@ -325,7 +325,7 @@ impl MachineState {
 
     // see 8.4.4 of Draft Technical Corrigendum 2.
     pub(super) fn check_keysort_errors(&self) -> CallResult {
-        let stub   = MachineError::functor_stub(clause_name!("keysort"), 2);        
+        let stub   = MachineError::functor_stub(clause_name!("keysort"), 2);
         let pairs  = self.store(self.deref(self[temp_v!(1)].clone()));
         let sorted = self.store(self.deref(self[temp_v!(2)].clone()));
 
@@ -366,3 +366,41 @@ impl MachineState {
         self.unwind_stack();
     }
 }
+
+pub enum SessionError {
+    CannotOverwriteBuiltIn(ClauseName),
+    CannotOverwriteImport(ClauseName),
+    ModuleDoesNotContainExport,
+    ModuleNotFound,
+    NamelessEntry,
+    OpIsInfixAndPostFix,
+    ParserError(ParserError),
+    QueryFailure,
+    QueryFailureWithException(ClauseName),
+    UserPrompt
+}
+
+pub enum EvalSession {
+    EntrySuccess,
+    Error(SessionError),
+    InitialQuerySuccess(AllocVarDict, HeapVarDict),
+    SubsequentQuerySuccess,
+}
+
+impl From<SessionError> for EvalSession {
+    fn from(err: SessionError) -> Self {
+        EvalSession::Error(err)
+    }
+}
+
+impl From<ParserError> for SessionError {
+    fn from(err: ParserError) -> Self {
+        SessionError::ParserError(err)
+    }
+}
+
+impl From<ParserError> for EvalSession {
+    fn from(err: ParserError) -> Self {
+        EvalSession::from(SessionError::ParserError(err))
+    }
+}

src/prolog/machine/machine_indices.rs

@@ -0,0 +1,533 @@
+use prolog_parser::ast::*;
+use prolog_parser::tabled_rc::*;
+
+use prolog::clause_types::*;
+use prolog::fixtures::*;
+use prolog::forms::*;
+
+use std::cell::RefCell;
+use std::cmp::Ordering;
+use std::collections::{HashMap, VecDeque};
+use std::mem;
+use std::ops::{Add, AddAssign, Sub, SubAssign};
+use std::rc::Rc;
+
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub enum Addr {
+    AttrVar(usize),
+    Con(Constant),
+    Lis(usize),
+    HeapCell(usize),
+    StackCell(usize, usize),
+    Str(usize)
+}
+
+#[derive(Clone, Copy, Hash, Eq, PartialEq)]
+pub enum Ref {
+    AttrVar(usize),
+    HeapCell(usize),
+    StackCell(usize, usize)
+}
+
+impl Ref {
+    pub fn as_addr(self) -> Addr {
+        match self {
+            Ref::AttrVar(h)        => Addr::AttrVar(h),
+            Ref::HeapCell(h)       => Addr::HeapCell(h),
+            Ref::StackCell(fr, sc) => Addr::StackCell(fr, sc)
+        }
+    }
+}
+
+impl PartialEq<Ref> for Addr {
+    fn eq(&self, r: &Ref) -> bool {
+        self.as_var() == Some(*r)
+    }
+}
+
+// for use in MachineState::bind.
+impl PartialOrd<Ref> for Addr {
+    fn partial_cmp(&self, r: &Ref) -> Option<Ordering> {
+        match self {
+            &Addr::StackCell(fr, sc) =>
+                match *r {
+                    Ref::AttrVar(_) | Ref::HeapCell(_) =>
+                        Some(Ordering::Greater),
+                    Ref::StackCell(fr1, sc1) =>
+                        if fr1 < fr || (fr1 == fr && sc1 < sc) {
+                            Some(Ordering::Greater)
+                        } else if fr1 == fr && sc1 == sc {
+                            Some(Ordering::Equal)
+                        } else {
+                            Some(Ordering::Less)
+                        }
+                },
+            &Addr::HeapCell(h) | &Addr::AttrVar(h) =>
+                match r {
+                    &Ref::StackCell(..) => Some(Ordering::Less),
+                    &Ref::AttrVar(h1) | &Ref::HeapCell(h1) => h.partial_cmp(&h1)
+                },
+            _ => None
+        }
+    }
+}
+
+impl Addr {
+    pub fn is_ref(&self) -> bool {
+        match self {
+            &Addr::AttrVar(_) | &Addr::HeapCell(_) | &Addr::StackCell(_, _) => true,
+            _ => false
+        }
+    }
+
+    pub fn as_var(&self) -> Option<Ref> {
+        match self {
+            &Addr::AttrVar(h) => Some(Ref::AttrVar(h)),
+            &Addr::HeapCell(h) => Some(Ref::HeapCell(h)),
+            &Addr::StackCell(fr, sc) => Some(Ref::StackCell(fr, sc)),
+            _ => None
+        }
+    }
+
+    pub fn is_protected(&self, e: usize) -> bool {
+        match self {
+            &Addr::StackCell(addr, _) if addr >= e => false,
+            _ => true
+        }
+    }
+}
+
+impl Add<usize> for Addr {
+    type Output = Addr;
+
+    fn add(self, rhs: usize) -> Self::Output {
+        match self {
+            Addr::Lis(a) => Addr::Lis(a + rhs),
+            Addr::AttrVar(h) => Addr::AttrVar(h + rhs),
+            Addr::HeapCell(h) => Addr::HeapCell(h + rhs),
+            Addr::Str(s) => Addr::Str(s + rhs),
+            _ => self
+        }
+    }
+}
+
+impl Sub<usize> for Addr {
+    type Output = Addr;
+
+    fn sub(self, rhs: usize) -> Self::Output {
+        match self {
+            Addr::Lis(a) => Addr::Lis(a - rhs),
+            Addr::AttrVar(h) => Addr::AttrVar(h - rhs),
+            Addr::HeapCell(h) => Addr::HeapCell(h - rhs),
+            Addr::Str(s) => Addr::Str(s - rhs),
+            _ => self
+        }
+    }
+}
+
+impl SubAssign<usize> for Addr {
+    fn sub_assign(&mut self, rhs: usize) {
+        *self = self.clone() - rhs;
+    }
+}
+
+impl From<Ref> for Addr {
+    fn from(r: Ref) -> Self {
+        match r {
+            Ref::AttrVar(h)        => Addr::AttrVar(h),
+            Ref::HeapCell(h)       => Addr::HeapCell(h),
+            Ref::StackCell(fr, sc) => Addr::StackCell(fr, sc)
+        }
+    }
+}
+
+#[derive(Clone)]
+pub enum TrailRef {
+    Ref(Ref),
+    AttrVarLink(usize, Addr)
+}
+
+impl From<Ref> for TrailRef {
+    fn from(r: Ref) -> Self {
+        TrailRef::Ref(r)
+    }
+}
+
+#[derive(Clone, PartialEq)]
+pub enum HeapCellValue {
+    Addr(Addr),
+    NamedStr(usize, ClauseName, Option<(usize, Specifier)>), // arity, name, precedence/Specifier if it has one.
+}
+
+impl HeapCellValue {
+    pub fn as_addr(&self, focus: usize) -> Addr {
+        match self {
+            &HeapCellValue::Addr(ref a) => a.clone(),
+            &HeapCellValue::NamedStr(_, _, _) => Addr::Str(focus)
+        }
+    }
+}
+
+#[derive(Clone, Copy, PartialEq)]
+pub enum IndexPtr {
+    Undefined,
+    Index(usize),
+}
+
+#[derive(Clone)]
+pub struct CodeIndex(pub Rc<RefCell<(IndexPtr, ClauseName)>>);
+
+impl CodeIndex {
+    #[inline]
+    pub fn is_undefined(&self) -> bool {
+        let index_ptr = &self.0.borrow().0;
+
+        if let &IndexPtr::Undefined = index_ptr {
+            true
+        } else {
+            false
+        }
+    }
+
+    #[inline]
+    pub fn module_name(&self) -> ClauseName {
+        self.0.borrow().1.clone()
+    }
+}
+
+impl Default for CodeIndex {
+    fn default() -> Self {
+        CodeIndex(Rc::new(RefCell::new((IndexPtr::Undefined, clause_name!("")))))
+    }
+}
+
+impl From<(usize, ClauseName)> for CodeIndex {
+    fn from(value: (usize, ClauseName)) -> Self {
+        CodeIndex(Rc::new(RefCell::new((IndexPtr::Index(value.0), value.1))))
+    }
+}
+
+#[derive(Clone, Copy, PartialEq)]
+pub enum DynamicAssertPlace {
+    Back, Front
+}
+
+impl DynamicAssertPlace {
+    #[inline]
+    pub fn predicate_name(self) -> ClauseName {
+        match self {
+            DynamicAssertPlace::Back  => clause_name!("assertz"),
+            DynamicAssertPlace::Front => clause_name!("asserta")
+        }
+    }
+
+    #[inline]
+    pub fn push_to_queue(self, addrs: &mut VecDeque<Addr>, new_addr: Addr) {
+        match self {
+            DynamicAssertPlace::Back  => addrs.push_back(new_addr),
+            DynamicAssertPlace::Front => addrs.push_front(new_addr)
+        }
+    }
+}
+
+#[derive(Clone, Copy, PartialEq)]
+pub enum DynamicTransactionType {
+    Abolish,
+    Assert(DynamicAssertPlace),
+    Retract // dynamic index of the clause to remove.
+}
+
+#[derive(Clone, PartialEq)]
+pub enum CodePtr {
+    BuiltInClause(BuiltInClauseType, LocalCodePtr), // local is the successor call.
+    CallN(usize, LocalCodePtr), // arity, local.
+    Local(LocalCodePtr),
+    DynamicTransaction(DynamicTransactionType, LocalCodePtr), // the type of transaction, the return pointer.
+    VerifyAttrInterrupt(usize), // location of the verify attribute interrupt code in the CodeDir.
+}
+
+impl CodePtr {
+    pub fn local(&self) -> LocalCodePtr {
+        match self {
+            &CodePtr::BuiltInClause(_, ref local)
+          | &CodePtr::CallN(_, ref local)
+          | &CodePtr::Local(ref local) => local.clone(),
+            &CodePtr::VerifyAttrInterrupt(p) => LocalCodePtr::DirEntry(p),
+            &CodePtr::DynamicTransaction(_, p) => p
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq)]
+pub enum LocalCodePtr {
+    DirEntry(usize), // offset.
+    InSituDirEntry(usize),
+    TopLevel(usize, usize), // chunk_num, offset.
+    UserGoalExpansion(usize),
+    UserTermExpansion(usize)
+}
+
+impl LocalCodePtr {
+    pub fn assign_if_local(&mut self, cp: CodePtr) {
+        match cp {
+            CodePtr::Local(local) => *self = local,
+            _ => {}
+        }
+    }
+}
+
+impl PartialOrd<CodePtr> for CodePtr {
+    fn partial_cmp(&self, other: &CodePtr) -> Option<Ordering> {
+        match (self, other) {
+            (&CodePtr::Local(ref l1), &CodePtr::Local(ref l2)) => l1.partial_cmp(l2),
+            _ => Some(Ordering::Greater)
+        }
+    }
+}
+
+impl PartialOrd<LocalCodePtr> for LocalCodePtr {
+    fn partial_cmp(&self, other: &LocalCodePtr) -> Option<Ordering> {
+        match (self, other) {
+            (&LocalCodePtr::InSituDirEntry(p1), &LocalCodePtr::InSituDirEntry(ref p2))
+          | (&LocalCodePtr::DirEntry(p1), &LocalCodePtr::DirEntry(ref p2))
+          | (&LocalCodePtr::UserTermExpansion(p1), &LocalCodePtr::UserTermExpansion(ref p2))
+          | (&LocalCodePtr::UserGoalExpansion(p1), &LocalCodePtr::UserGoalExpansion(ref p2))
+          | (&LocalCodePtr::TopLevel(_, p1), &LocalCodePtr::TopLevel(_, ref p2)) =>
+                p1.partial_cmp(p2),
+            (_, &LocalCodePtr::TopLevel(_, _)) =>
+                Some(Ordering::Less),
+            _ => Some(Ordering::Greater)
+        }
+    }
+}
+
+impl Default for CodePtr {
+    fn default() -> Self {
+        CodePtr::Local(LocalCodePtr::default())
+    }
+}
+
+impl Default for LocalCodePtr {
+    fn default() -> Self {
+        LocalCodePtr::TopLevel(0, 0)
+    }
+}
+
+impl Add<usize> for LocalCodePtr {
+    type Output = LocalCodePtr;
+
+    fn add(self, rhs: usize) -> Self::Output {
+        match self {
+            LocalCodePtr::InSituDirEntry(p) => LocalCodePtr::InSituDirEntry(p + rhs),
+            LocalCodePtr::DirEntry(p) => LocalCodePtr::DirEntry(p + rhs),
+            LocalCodePtr::TopLevel(cn, p) => LocalCodePtr::TopLevel(cn, p + rhs),
+            LocalCodePtr::UserTermExpansion(p) => LocalCodePtr::UserTermExpansion(p + rhs),
+            LocalCodePtr::UserGoalExpansion(p) => LocalCodePtr::UserGoalExpansion(p + rhs),
+        }
+    }
+}
+
+impl AddAssign<usize> for LocalCodePtr {
+    fn add_assign(&mut self, rhs: usize) {
+        match self {
+            &mut LocalCodePtr::InSituDirEntry(ref mut p)
+          | &mut LocalCodePtr::UserGoalExpansion(ref mut p)
+          | &mut LocalCodePtr::UserTermExpansion(ref mut p)
+          | &mut LocalCodePtr::DirEntry(ref mut p)
+          | &mut LocalCodePtr::TopLevel(_, ref mut p) => *p += rhs
+        }
+    }
+}
+
+impl Add<usize> for CodePtr {
+    type Output = CodePtr;
+
+    fn add(self, rhs: usize) -> Self::Output {
+        match self {
+            p @ CodePtr::VerifyAttrInterrupt(_)
+          | p @ CodePtr::DynamicTransaction(..) => p,
+            CodePtr::Local(local) => CodePtr::Local(local + rhs),
+            CodePtr::CallN(_, local) | CodePtr::BuiltInClause(_, local) => CodePtr::Local(local + rhs)
+        }
+    }
+}
+
+impl AddAssign<usize> for CodePtr {
+    fn add_assign(&mut self, rhs: usize) {
+        match self {
+            &mut CodePtr::VerifyAttrInterrupt(_) => {},
+            &mut CodePtr::Local(ref mut local) => *local += rhs,
+            _ => *self = CodePtr::Local(self.local() + rhs)
+        }
+    }
+}
+
+pub type HeapVarDict  = HashMap<Rc<Var>, Addr>;
+pub type AllocVarDict = HashMap<Rc<Var>, VarData>;
+
+#[derive(Copy, Clone)]
+pub struct DynamicPredicateInfo {
+    pub(super) clauses_subsection_p: usize, // a LocalCodePtr::DirEntry value.
+}
+
+impl Default for DynamicPredicateInfo {
+    fn default() -> Self {
+        DynamicPredicateInfo { clauses_subsection_p: 0 }
+    }
+}
+
+pub type InSituCodeDir  = HashMap<PredicateKey, usize>;
+pub type DynamicCodeDir = HashMap<PredicateKey, DynamicPredicateInfo>;
+
+pub struct IndexStore {
+    pub(super) atom_tbl: TabledData<Atom>,
+    pub(super) code_dir: CodeDir,
+    pub(super) dynamic_code_dir: DynamicCodeDir,
+    pub(super) in_situ_code_dir: InSituCodeDir,
+    pub(super) op_dir: OpDir,
+    pub(super) modules: ModuleDir,
+}
+
+impl IndexStore {
+    pub fn predicate_exists(&self, name: ClauseName, arity: usize,
+                            op_spec: Option<(usize, Specifier)>)
+                            -> bool
+    {
+        match ClauseType::from(name, arity, op_spec) {
+            ClauseType::Named(name, arity, _) =>
+                self.code_dir.contains_key(&(name, arity)),
+            ClauseType::Op(op_decl, ..) =>
+                self.code_dir.contains_key(&(op_decl.name(), op_decl.arity())),
+            _ => true
+        }
+    }
+
+    #[inline]
+    pub fn get_clause_subsection(&self, name: ClauseName, arity: usize) -> Option<DynamicPredicateInfo> {
+        self.dynamic_code_dir.get(&(name, arity)).cloned()
+    }
+
+    #[inline]
+    pub fn take_module(&mut self, name: ClauseName) -> Option<Module> {
+        self.modules.remove(&name)
+    }
+
+    #[inline]
+    pub fn insert_module(&mut self, module: Module) {
+        self.modules.insert(module.module_decl.name.clone(), module);
+    }
+
+    #[inline]
+    pub(super) fn new() -> Self {
+        IndexStore {
+            atom_tbl: TabledData::new(Rc::new("user".to_string())),
+            code_dir: CodeDir::new(),
+            dynamic_code_dir: DynamicCodeDir::new(),
+            in_situ_code_dir: InSituCodeDir::new(),
+            op_dir: default_op_dir(),
+            modules: ModuleDir::new(),
+        }
+    }
+
+    #[inline]
+    pub(super) fn copy_and_swap(&mut self, other: &mut IndexStore) {
+        self.code_dir = other.code_dir.clone();
+        self.op_dir = other.op_dir.clone();
+
+        mem::swap(&mut self.code_dir, &mut other.code_dir);
+        mem::swap(&mut self.op_dir, &mut other.op_dir);
+        mem::swap(&mut self.modules, &mut other.modules);
+    }
+
+    #[inline]
+    fn get_internal(&self, name: ClauseName, arity: usize, in_mod: ClauseName)
+                    -> Option<ModuleCodeIndex>
+    {
+        self.modules.get(&in_mod)
+            .and_then(|ref module| module.code_dir.get(&(name, arity)))
+            .cloned()
+    }
+
+    pub(super) fn get_cleaner_sites(&self) -> (usize, usize) {
+        let r_w_h  = clause_name!("run_cleaners_with_handling");
+        let r_wo_h = clause_name!("run_cleaners_without_handling");
+
+        let builtins = clause_name!("builtins");
+
+        let r_w_h  = self.get_internal(r_w_h, 0, builtins.clone()).and_then(|item| item.local());
+        let r_wo_h = self.get_internal(r_wo_h, 1, builtins).and_then(|item| item.local());
+
+        if let Some(r_w_h) = r_w_h {
+            if let Some(r_wo_h) = r_wo_h {
+                return (r_w_h, r_wo_h);
+            }
+        }
+
+        return (0, 0);
+    }
+}
+
+pub type CodeDir = HashMap<PredicateKey, CodeIndex>;
+pub type TermDir = HashMap<PredicateKey, (Predicate, VecDeque<TopLevel>)>;
+
+#[derive(Clone, Copy)]
+pub enum CompileTimeHook {
+    GoalExpansion,
+    TermExpansion,
+    UserGoalExpansion,
+    UserTermExpansion
+}
+
+impl CompileTimeHook {
+    pub fn name(self) -> ClauseName {
+        match self {
+            CompileTimeHook::UserGoalExpansion
+          | CompileTimeHook::GoalExpansion => clause_name!("goal_expansion"),
+            CompileTimeHook::UserTermExpansion
+          | CompileTimeHook::TermExpansion => clause_name!("term_expansion")
+        }
+    }
+
+    #[inline]
+    pub fn arity(self) -> usize {
+        match self {
+            CompileTimeHook::UserGoalExpansion
+          | CompileTimeHook::GoalExpansion => 2,
+            CompileTimeHook::UserTermExpansion
+          | CompileTimeHook::TermExpansion => 2
+        }
+    }
+
+    #[inline]
+    pub fn user_scope(self) -> Self {
+        match self {
+            CompileTimeHook::UserGoalExpansion | CompileTimeHook::GoalExpansion =>
+                CompileTimeHook::UserGoalExpansion,
+            CompileTimeHook::UserTermExpansion | CompileTimeHook::TermExpansion =>
+                CompileTimeHook::UserTermExpansion,
+        }
+    }
+
+    #[inline]
+    pub fn has_module_scope(self) -> bool {
+        match self {
+            CompileTimeHook::UserTermExpansion | CompileTimeHook::UserGoalExpansion => false,
+            _ => true
+        }
+    }
+}
+
+pub(super) enum RefOrOwned<'a, T: 'a> {
+    Borrowed(&'a T),
+    Owned(T)
+}
+
+impl<'a, T> RefOrOwned<'a, T> {
+    pub(super)
+    fn as_ref(&'a self) -> &'a T {
+        match self {
+            &RefOrOwned::Borrowed(r) => r,
+            &RefOrOwned::Owned(ref r) => r
+        }
+    }
+}

src/prolog/machine/machine_state.rs

@@ -1,14 +1,17 @@
 use prolog_parser::ast::*;
 use prolog_parser::string_list::*;
 
-use prolog::instructions::*;
-use prolog::and_stack::*;
-use prolog::copier::*;
-use prolog::heap::*;
-use prolog::machine::{AttrVarInitializer, IndexStore};
+use prolog::clause_types::*;
+use prolog::forms::*;
+use prolog::machine::and_stack::*;
+use prolog::machine::attributed_variables::*;
+use prolog::machine::copier::*;
+use prolog::machine::heap::*;
 use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
+use prolog::machine::modules::*;
+use prolog::machine::or_stack::*;
 use prolog::num::{BigInt, BigUint, Zero, One};
-use prolog::or_stack::*;
 
 use downcast::Any;
 
@@ -188,6 +191,8 @@ impl IndexMut<RegType> for MachineState {
     }
 }
 
+pub type Registers = Vec<Addr>;
+
 #[derive(Clone, Copy)]
 pub(super) enum MachineMode {
     Read,

src/prolog/machine/machine_state_impl.rs

@@ -1,19 +1,22 @@
 use prolog_parser::ast::*;
 use prolog_parser::string_list::StringList;
 
-use prolog::instructions::*;
-use prolog::and_stack::*;
-use prolog::copier::*;
-use prolog::heap::*;
+use prolog::clause_types::*;
+use prolog::forms::*;
 use prolog::heap_iter::*;
 use prolog::heap_print::*;
-use prolog::machine::{AttrVarInitializer, IndexStore};
+use prolog::instructions::*;
+use prolog::machine::attributed_variables::*;
+use prolog::machine::and_stack::*;
+use prolog::machine::copier::*;
+use prolog::machine::heap::*;
+use prolog::machine::or_stack::*;
 use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
 use prolog::machine::machine_state::*;
 use prolog::num::{Integer, Signed, ToPrimitive, Zero};
 use prolog::num::bigint::{BigInt, BigUint};
 use prolog::num::rational::Ratio;
-use prolog::or_stack::*;
 
 use std::cmp::{max, Ordering};
 use std::collections::{HashMap, HashSet};

src/prolog/machine/mod.rs

@@ -1,15 +1,24 @@
 use prolog_parser::ast::*;
 use prolog_parser::tabled_rc::*;
 
-use prolog::codegen::*;
-use prolog::compile::*;
-use prolog::debray_allocator::*;
+use prolog::clause_types::*;
+use prolog::fixtures::*;
+use prolog::forms::*;
 use prolog::heap_print::*;
 use prolog::instructions::*;
 
+pub mod machine_indices;
+pub mod heap;
+mod and_stack;
+mod or_stack;
 mod attributed_variables;
+mod copier;
 mod dynamic_database;
-mod machine_errors;
+pub mod machine_errors;
+pub mod toplevel;
+pub mod compile;
+pub(super) mod code_repo;
+pub mod modules;
 pub(super) mod machine_state;
 pub(super) mod term_expansion;
 
@@ -17,233 +26,18 @@ pub(super) mod term_expansion;
 mod system_calls;
 
 use prolog::machine::attributed_variables::*;
+use prolog::machine::compile::*;
+use prolog::machine::code_repo::*;
+use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
 use prolog::machine::machine_state::*;
+use prolog::machine::modules::*;
 
 use std::collections::{HashMap, VecDeque};
 use std::mem;
 use std::ops::Index;
 use std::rc::Rc;
 
-
-#[derive(Copy, Clone)]
-pub struct DynamicPredicateInfo {
-    pub(super) clauses_subsection_p: usize, // a LocalCodePtr::DirEntry value.
-}
-
-impl Default for DynamicPredicateInfo {
-    fn default() -> Self {
-        DynamicPredicateInfo { clauses_subsection_p: 0 }
-    }
-}
-
-pub type InSituCodeDir  = HashMap<PredicateKey, usize>;
-pub type DynamicCodeDir = HashMap<PredicateKey, DynamicPredicateInfo>;
-
-pub struct IndexStore {
-    pub(super) atom_tbl: TabledData<Atom>,
-    pub(super) code_dir: CodeDir,
-    pub(super) dynamic_code_dir: DynamicCodeDir,
-    pub(super) in_situ_code_dir: InSituCodeDir,
-    pub(super) op_dir: OpDir,
-    pub(super) modules: ModuleDir,
-}
-
-enum RefOrOwned<'a, T: 'a> {
-    Borrowed(&'a T),
-    Owned(T)
-}
-
-impl<'a, T> RefOrOwned<'a, T> {
-    fn as_ref(&'a self) -> &'a T {
-        match self {
-            &RefOrOwned::Borrowed(r) => r,
-            &RefOrOwned::Owned(ref r) => r
-        }
-    }
-}
-
-impl IndexStore {
-    pub fn predicate_exists(&self, name: ClauseName, arity: usize,
-                            op_spec: Option<(usize, Specifier)>)
-                            -> bool
-    {
-        match ClauseType::from(name, arity, op_spec) {
-            ClauseType::Named(name, arity, _) =>
-                self.code_dir.contains_key(&(name, arity)),
-            ClauseType::Op(op_decl, ..) =>
-                self.code_dir.contains_key(&(op_decl.name(), op_decl.arity())),
-            _ => true
-        }
-    }
-
-    #[inline]
-    pub fn get_clause_subsection(&self, name: ClauseName, arity: usize) -> Option<DynamicPredicateInfo> {
-        self.dynamic_code_dir.get(&(name, arity)).cloned()
-    }
-
-    #[inline]
-    pub fn take_module(&mut self, name: ClauseName) -> Option<Module> {
-        self.modules.remove(&name)
-    }
-
-    #[inline]
-    pub fn insert_module(&mut self, module: Module) {
-        self.modules.insert(module.module_decl.name.clone(), module);
-    }
-
-    #[inline]
-    pub(super) fn new() -> Self {
-        IndexStore {
-            atom_tbl: TabledData::new(Rc::new("user".to_string())),
-            code_dir: CodeDir::new(),
-            dynamic_code_dir: DynamicCodeDir::new(),
-            in_situ_code_dir: InSituCodeDir::new(),
-            op_dir: default_op_dir(),
-            modules: ModuleDir::new(),
-        }
-    }
-
-    #[inline]
-    pub(super) fn copy_and_swap(&mut self, other: &mut IndexStore) {
-        self.code_dir = other.code_dir.clone();
-        self.op_dir = other.op_dir.clone();
-
-        mem::swap(&mut self.code_dir, &mut other.code_dir);
-        mem::swap(&mut self.op_dir, &mut other.op_dir);
-        mem::swap(&mut self.modules, &mut other.modules);
-    }
-
-    #[inline]
-    fn get_internal(&self, name: ClauseName, arity: usize, in_mod: ClauseName)
-                    -> Option<ModuleCodeIndex>
-    {
-        self.modules.get(&in_mod)
-            .and_then(|ref module| module.code_dir.get(&(name, arity)))
-            .cloned()
-    }
-
-    pub(super) fn get_cleaner_sites(&self) -> (usize, usize) {
-        let r_w_h  = clause_name!("run_cleaners_with_handling");
-        let r_wo_h = clause_name!("run_cleaners_without_handling");
-
-        let builtins = clause_name!("builtins");
-
-        let r_w_h  = self.get_internal(r_w_h, 0, builtins.clone()).and_then(|item| item.local());
-        let r_wo_h = self.get_internal(r_wo_h, 1, builtins).and_then(|item| item.local());
-
-        if let Some(r_w_h) = r_w_h {
-            if let Some(r_wo_h) = r_wo_h {
-                return (r_w_h, r_wo_h);
-            }
-        }
-
-        return (0, 0);
-    }
-}
-
-pub type CompiledResult = (Predicate, VecDeque<TopLevel>);
-
-impl CodeRepo {
-    #[inline]
-    fn new() -> Self {
-        CodeRepo {
-            cached_query: vec![],
-            goal_expanders: Code::new(),
-            term_expanders: Code::new(),
-            code: Code::new(),
-            in_situ_code: Code::new(),
-            term_dir: TermDir::new()
-        }
-    }
-
-    #[inline]
-    pub fn term_dir_entry_len(&self, key: PredicateKey) -> (usize, usize) {
-        self.term_dir.get(&key)
-            .map(|entry| ((entry.0).0.len(), entry.1.len()))
-            .unwrap_or((0,0))
-    }
-
-    #[inline]
-    pub fn truncate_terms(&mut self, key: PredicateKey, len: usize, queue_len: usize)
-                          -> (Predicate, VecDeque<TopLevel>)
-    {
-        self.term_dir.get_mut(&key)
-            .map(|entry| (Predicate((entry.0).0.drain(len ..).collect()),
-                          entry.1.drain(queue_len ..).collect()))
-            .unwrap_or((Predicate::new(), VecDeque::from(vec![])))
-    }
-
-    pub fn add_in_situ_result(&mut self, result: &CompiledResult, in_situ_code_dir: &mut InSituCodeDir,
-                              flags: MachineFlags)
-                              -> Result<(), SessionError>
-    {
-        let (ref decl, ref queue) = result;
-        let (name, arity) = decl.0.first().and_then(|cl| {
-            let arity = cl.arity();
-            cl.name().map(|name| (name, arity))
-        }).ok_or(SessionError::NamelessEntry)?;
-
-        let p = self.in_situ_code.len();
-        in_situ_code_dir.insert((name, arity), p);
-
-        let mut cg = CodeGenerator::<DebrayAllocator>::new(true, flags);
-        // clone the decl to avoid the need to wipe its register cells later.
-        let mut decl_code = cg.compile_predicate(&decl.0.clone())?;
-
-        compile_appendix(&mut decl_code, queue, true, flags)?;
-
-        self.in_situ_code.extend(decl_code.into_iter());
-        Ok(())
-    }
-
-    #[inline]
-    fn size_of_cached_query(&self) -> usize {
-        self.cached_query.len()
-    }
-
-    fn lookup_instr<'a>(&'a self, last_call: bool, p: &CodePtr) -> Option<RefOrOwned<'a, Line>>
-    {
-        match p {
-            &CodePtr::Local(LocalCodePtr::UserGoalExpansion(p)) =>
-                if p < self.goal_expanders.len() {
-                    Some(RefOrOwned::Borrowed(&self.goal_expanders[p]))
-                } else {
-                    None
-                },
-            &CodePtr::Local(LocalCodePtr::UserTermExpansion(p)) =>
-                if p < self.term_expanders.len() {
-                    Some(RefOrOwned::Borrowed(&self.term_expanders[p]))
-                } else {
-                    None
-                },
-            &CodePtr::Local(LocalCodePtr::TopLevel(_, p)) =>
-                if p < self.cached_query.len() {
-                    Some(RefOrOwned::Borrowed(&self.cached_query[p]))
-                } else {
-                    None
-                },
-            &CodePtr::Local(LocalCodePtr::InSituDirEntry(p)) =>
-                Some(RefOrOwned::Borrowed(&self.in_situ_code[p])),
-            &CodePtr::Local(LocalCodePtr::DirEntry(p)) =>
-                Some(RefOrOwned::Borrowed(&self.code[p])),
-            &CodePtr::BuiltInClause(ref built_in, _) => {
-                let call_clause = call_clause!(ClauseType::BuiltIn(built_in.clone()),
-                                               built_in.arity(),
-                                               0, last_call);
-                Some(RefOrOwned::Owned(call_clause))
-            },
-            &CodePtr::CallN(arity, _) => {
-                let call_clause = call_clause!(ClauseType::CallN, arity, 0, last_call);
-                Some(RefOrOwned::Owned(call_clause))
-            },
-            &CodePtr::VerifyAttrInterrupt(p) =>
-                Some(RefOrOwned::Borrowed(&self.code[p])),
-            &CodePtr::DynamicTransaction(..) =>
-                None
-        }
-    }
-}
-
 pub struct MachinePolicies {
     call_policy: Box<CallPolicy>,
     cut_policy: Box<CutPolicy>,

src/prolog/machine/modules.rs

@@ -0,0 +1,242 @@
+use prolog_parser::ast::*;
+use prolog_parser::tabled_rc::*;
+
+use prolog::forms::*;
+use prolog::machine::code_repo::*;
+use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
+
+use std::cell::RefCell;
+use std::collections::{VecDeque};
+use std::rc::Rc;
+
+impl ModuleCodeIndex {
+    pub fn local(&self) -> Option<usize> {
+        match self.0 {
+            IndexPtr::Index(i) => Some(i),
+            _ => None
+        }
+    }
+}
+
+impl From<ModuleCodeIndex> for CodeIndex {
+    fn from(value: ModuleCodeIndex) -> Self {
+        CodeIndex(Rc::new(RefCell::new((value.0, value.1))))
+    }
+}
+
+// Module's and related types are defined in forms.
+impl Module {
+    pub fn new(module_decl: ModuleDecl, atom_tbl: TabledData<Atom>) -> Self {
+        Module { module_decl, atom_tbl,
+                 term_expansions: (Predicate::new(), VecDeque::from(vec![])),
+                 goal_expansions: (Predicate::new(), VecDeque::from(vec![])),
+                 code_dir: ModuleCodeDir::new(),
+                 op_dir: default_op_dir(),
+                 inserted_expansions: false }
+    }
+
+    pub fn dump_expansions(&self, code_repo: &mut CodeRepo, flags: MachineFlags)
+                           -> Result<(), ParserError>
+    {
+        {
+            let te = code_repo.term_dir.entry((clause_name!("term_expansion"), 2))
+                .or_insert((Predicate::new(), VecDeque::from(vec![])));
+
+            (te.0).0.extend((self.term_expansions.0).0.iter().cloned());
+            te.1.extend(self.term_expansions.1.iter().cloned());
+        }
+
+        {
+            let ge = code_repo.term_dir.entry((clause_name!("goal_expansion"), 2))
+                .or_insert((Predicate::new(), VecDeque::from(vec![])));
+
+            (ge.0).0.extend((self.goal_expansions.0).0.iter().cloned());
+            ge.1.extend(self.goal_expansions.1.iter().cloned());
+        }
+
+        code_repo.compile_hook(CompileTimeHook::TermExpansion, flags)?;
+        code_repo.compile_hook(CompileTimeHook::GoalExpansion, flags)?;
+
+        Ok(())
+    }
+}
+
+pub trait SubModuleUser
+{
+    fn atom_tbl(&self) -> TabledData<Atom>;
+    fn op_dir(&mut self) -> &mut OpDir;
+    fn remove_code_index(&mut self, PredicateKey);
+    fn get_code_index(&self, PredicateKey, ClauseName) -> Option<CodeIndex>;
+
+    fn insert_dir_entry(&mut self, ClauseName, usize, ModuleCodeIndex);
+
+    fn remove_module(&mut self, mod_name: ClauseName, module: &Module) {
+        for (name, arity) in module.module_decl.exports.iter().cloned() {
+            let name = name.defrock_brackets();
+
+            match self.get_code_index((name.clone(), arity), mod_name.clone()) {
+                Some(CodeIndex (ref code_idx)) => {
+                    if &code_idx.borrow().1 != &module.module_decl.name {
+                        continue;
+                    }
+
+                    self.remove_code_index((name.clone(), arity));
+
+                    // remove or respecify ops.
+                    if arity == 2 {
+                        if let Some((_, _, mod_name)) = self.op_dir().get(&(name.clone(), Fixity::In)).cloned()
+                        {
+                            if mod_name == module.module_decl.name {
+                                self.op_dir().remove(&(name.clone(), Fixity::In));
+                            }
+                        }
+                    } else if arity == 1 {
+                        if let Some((_, _, mod_name)) = self.op_dir().get(&(name.clone(), Fixity::Pre)).cloned()
+                        {
+                            if mod_name == module.module_decl.name {
+                                self.op_dir().remove(&(name.clone(), Fixity::Pre));
+                            }
+                        }
+
+                        if let Some((_, _, mod_name)) = self.op_dir().get(&(name.clone(), Fixity::Post)).cloned()
+                        {
+                            if mod_name == module.module_decl.name {
+                                self.op_dir().remove(&(name.clone(), Fixity::Post));
+                            }
+                        }
+                    }
+                },
+                _ => {}
+            };
+        }
+    }
+
+    // returns true on successful import.
+    fn import_decl(&mut self, name: ClauseName, arity: usize, submodule: &Module) -> bool {
+        let name = name.defrock_brackets();
+        let mut found_op = false;
+
+        {
+            let mut insert_op_dir = |fix| {
+                if let Some(op_data) = submodule.op_dir.get(&(name.clone(), fix)) {
+                    self.op_dir().insert((name.clone(), fix), op_data.clone());
+                    found_op = true;
+                }
+            };
+
+            if arity == 1 {
+                insert_op_dir(Fixity::Pre);
+                insert_op_dir(Fixity::Post);
+            } else if arity == 2 {
+                insert_op_dir(Fixity::In);
+            }
+        }
+
+        if let Some(code_data) = submodule.code_dir.get(&(name.clone(), arity)) {
+            let name = name.with_table(submodule.atom_tbl.clone());
+
+            let mut atom_tbl = self.atom_tbl();
+            atom_tbl.borrow_mut().insert(name.to_rc());
+
+            self.insert_dir_entry(name, arity, code_data.clone());
+            true
+        } else {
+            found_op
+        }
+    }
+
+    fn use_qualified_module(&mut self, &mut CodeRepo, MachineFlags, &Module, &Vec<PredicateKey>)
+                            -> Result<(), SessionError>;
+    fn use_module(&mut self, &mut CodeRepo, MachineFlags, &Module)
+                  -> Result<(), SessionError>;
+}
+
+pub fn use_qualified_module<User>(user: &mut User, submodule: &Module, exports: &Vec<PredicateKey>)
+                              -> Result<(), SessionError>
+  where User: SubModuleUser
+{
+    for (name, arity) in exports.iter().cloned() {
+        if !submodule.module_decl.exports.contains(&(name.clone(), arity)) {
+            continue;
+        }
+
+        if !user.import_decl(name, arity, submodule) {
+            return Err(SessionError::ModuleDoesNotContainExport);
+        }
+    }
+
+    Ok(())
+}
+
+pub fn use_module<User: SubModuleUser>(user: &mut User, submodule: &Module)
+                                       -> Result<(), SessionError>
+{
+    for (name, arity) in submodule.module_decl.exports.iter().cloned() {
+        if !user.import_decl(name, arity, submodule) {
+            return Err(SessionError::ModuleDoesNotContainExport);
+        }
+    }
+
+    Ok(())
+}
+
+impl SubModuleUser for Module {
+    fn atom_tbl(&self) -> TabledData<Atom> {
+        self.atom_tbl.clone()
+    }
+
+    fn op_dir(&mut self) -> &mut OpDir {
+        &mut self.op_dir
+    }
+
+    fn get_code_index(&self, key: PredicateKey, _: ClauseName) -> Option<CodeIndex> {
+        self.code_dir.get(&key).cloned().map(CodeIndex::from)
+    }
+
+    fn remove_code_index(&mut self, key: PredicateKey) {
+        self.code_dir.remove(&key);
+    }
+
+    fn insert_dir_entry(&mut self, name: ClauseName, arity: usize, idx: ModuleCodeIndex) {
+        self.code_dir.insert((name, arity), idx);
+    }
+
+    fn use_qualified_module(&mut self, _: &mut CodeRepo, _: MachineFlags, submodule: &Module,
+                            exports: &Vec<PredicateKey>)
+                            -> Result<(), SessionError>
+    {
+        use_qualified_module(self, submodule, exports)?;
+
+        (self.term_expansions.0).0.extend((submodule.term_expansions.0).0.iter().cloned());
+        self.term_expansions.1.extend(submodule.term_expansions.1.iter().cloned());
+
+        (self.goal_expansions.0).0.extend((submodule.goal_expansions.0).0.iter().cloned());
+        self.goal_expansions.1.extend(submodule.goal_expansions.1.iter().cloned());
+
+        Ok(())
+    }
+
+    fn use_module(&mut self, _: &mut CodeRepo, _: MachineFlags, submodule: &Module)
+                  -> Result<(), SessionError>
+    {
+        use_module(self, submodule)?;
+
+        (self.term_expansions.0).0.extend((submodule.term_expansions.0).0.iter().cloned());
+        self.term_expansions.1.extend(submodule.term_expansions.1.iter().cloned());
+
+        (self.goal_expansions.0).0.extend((submodule.goal_expansions.0).0.iter().cloned());
+        self.goal_expansions.1.extend(submodule.goal_expansions.1.iter().cloned());
+
+        Ok(())
+    }
+}
+
+pub fn as_module_code_dir(code_dir: CodeDir) -> ModuleCodeDir {
+    code_dir.into_iter()
+        .map(|(k, code_idx)| {
+            let (idx, module_name) = code_idx.0.borrow().clone();
+            (k, ModuleCodeIndex(idx, module_name))
+        })
+        .collect()
+}

src/prolog/machine/or_stack.rs

@@ -0,0 +1,126 @@
+use prolog::machine::machine_indices::*;
+
+use std::ops::{Index, IndexMut};
+use std::vec::Vec;
+
+pub struct Frame {
+    pub global_index: usize,
+    pub e: usize,
+    pub cp: LocalCodePtr,
+    pub attr_var_init_b: usize,
+    pub b: usize,    
+    pub bp: CodePtr,
+    pub tr: usize,
+    pub pstr_tr: usize,
+    pub h: usize,
+    pub b0: usize,
+    args: Vec<Addr>
+}
+
+impl Frame {
+    fn new(global_index: usize,
+           e: usize,
+           cp: LocalCodePtr,
+           attr_var_init_b: usize,
+           b: usize,
+           bp: CodePtr,
+           tr: usize,
+           pstr_tr: usize,
+           h: usize,
+           b0: usize,
+           n: usize)
+           -> Self
+    {
+        Frame {
+            global_index,
+            e,
+            cp,
+            attr_var_init_b,
+            b,
+            bp,
+            tr,
+            pstr_tr,
+            h,
+            b0,
+            args: vec![Addr::HeapCell(0); n]
+        }
+    }
+
+    pub fn num_args(&self) -> usize {
+        self.args.len()
+    }
+}
+
+pub struct OrStack(Vec<Frame>);
+
+impl OrStack {
+    pub fn new() -> Self {
+        OrStack(Vec::new())
+    }
+
+    pub fn push(&mut self,
+                global_index: usize,
+                e: usize,
+                cp: LocalCodePtr,
+                attr_var_init_b: usize,
+                b: usize,
+                bp: CodePtr,
+                tr: usize,
+                pstr_tr: usize,
+                h: usize,
+                b0: usize,
+                n: usize)
+    {
+        self.0.push(Frame::new(global_index, e, cp, attr_var_init_b, b, bp, tr, pstr_tr, h, b0, n));
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    pub fn clear(&mut self) {
+        self.0.clear()
+    }
+    
+    pub fn top(&self) -> Option<&Frame> {
+        self.0.last()
+    }
+
+    // truncate expects a 1-indexed new_b, ie.
+    // the value b of MachineState.
+    pub fn truncate(&mut self, new_b: usize) {
+        self.0.truncate(new_b);
+    }
+    
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+}
+
+impl Index<usize> for OrStack {
+    type Output = Frame;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        self.0.index(index)
+    }
+}
+
+impl IndexMut<usize> for OrStack {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        self.0.index_mut(index)
+    }
+}
+
+impl Index<usize> for Frame {
+    type Output = Addr;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        self.args.index(index - 1)
+    }
+}
+
+impl IndexMut<usize> for Frame {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        self.args.index_mut(index - 1)
+    }
+}

src/prolog/machine/system_calls.rs

@@ -1,11 +1,11 @@
 use prolog_parser::ast::*;
 
-use prolog::copier::*;
+use prolog::clause_types::*;
 use prolog::heap_iter::*;
 use prolog::heap_print::*;
-use prolog::instructions::*;
-use prolog::machine::IndexStore;
+use prolog::machine::copier::*;
 use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
 use prolog::machine::machine_state::*;
 use prolog::num::{ToPrimitive, Zero};
 use prolog::num::bigint::{BigInt};

src/prolog/machine/term_expansion.rs

@@ -1,8 +1,8 @@
 use prolog_parser::ast::*;
 use prolog_parser::parser::*;
 
-use prolog::instructions::HeapCellValue;
 use prolog::machine::*;
+use prolog::machine::machine_indices::HeapCellValue;
 use prolog::num::*;
 use prolog::read::*;
 

src/prolog/machine/toplevel.rs

@@ -0,0 +1,887 @@
+use prolog_parser::ast::*;
+use prolog_parser::tabled_rc::*;
+
+use prolog::forms::*;
+use prolog::iterators::*;
+use prolog::machine::*;
+use prolog::machine::code_repo::*;
+use prolog::machine::machine_errors::*;
+use prolog::machine::machine_indices::*;
+use prolog::machine::machine_state::MachineState;
+use prolog::machine::term_expansion::*;
+use prolog::num::*;
+
+use std::borrow::BorrowMut;
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::cell::Cell;
+use std::io::Read;
+use std::mem;
+use std::rc::Rc;
+
+struct CompositeIndices<'a, 'b> {
+    local: &'a mut IndexStore,
+    static_code_dir: Option<&'b CodeDir>
+}
+
+macro_rules! composite_indices {
+    ($in_module: expr, $local: expr, $static_code_dir: expr) => (
+        CompositeIndices { local: $local,
+                           static_code_dir: if $in_module {
+                               None
+                           } else {
+                               Some($static_code_dir)
+                           }}
+    );
+    ($local: expr) => (
+        CompositeIndices { local: $local, static_code_dir: None }
+    )
+}
+
+impl<'a, 'b> CompositeIndices<'a, 'b>
+{
+    fn get_code_index(&mut self, name: ClauseName, arity: usize) -> CodeIndex {
+        let idx_opt = self.local.code_dir.get(&(name.clone(), arity))
+            .or_else(|| {
+                match &self.static_code_dir {
+                    &Some(ref code_dir) => code_dir.get(&(name.clone(), arity)),
+                    _ => None
+                }
+            }).cloned();
+
+        if let Some(idx) = idx_opt {
+            self.local.code_dir.insert((name, arity), idx.clone());
+            idx
+        } else {
+            let idx = CodeIndex::default();
+            self.local.code_dir.insert((name, arity), idx.clone());
+            idx
+        }
+    }
+
+    fn get_clause_type(&mut self, name: ClauseName, arity: usize, spec: Option<(usize, Specifier)>) -> ClauseType
+    {
+        match ClauseType::from(name, arity, spec) {
+            ClauseType::Named(name, arity, _) => {
+                let idx = self.get_code_index(name.clone(), arity);
+                ClauseType::Named(name, arity, idx.clone())
+            },
+            ClauseType::Op(op_decl, _) => {
+                let idx = self.get_code_index(op_decl.2.clone(), arity);
+                ClauseType::Op(op_decl, idx.clone())
+            },
+            ct => ct
+        }
+    }
+}
+
+fn as_compile_time_hook(name: &str, arity: usize, terms: &Vec<Box<Term>>) -> Option<CompileTimeHook>
+{
+    match (name, arity) {
+        ("term_expansion", 2) => Some(CompileTimeHook::TermExpansion),
+        ("goal_expansion", 2) => Some(CompileTimeHook::GoalExpansion),
+        (":", 2) => {
+            if let &Term::Constant(_, Constant::Atom(ref name, _)) = &terms[0].as_ref() {
+                if name.as_str() == "user" {
+                    if let &Term::Clause(_, ref name, ref terms, _) = &terms[1].as_ref() {
+                        return match name.as_str() {
+                            "term_expansion" if terms.len() == 2 =>
+                                Some(CompileTimeHook::UserTermExpansion),
+                            "goal_expansion" if terms.len() == 2 =>
+                                Some(CompileTimeHook::UserGoalExpansion),
+                            _ => None
+                        }
+                    }
+                }
+            }
+
+            None
+        },
+        _ => None
+    }
+}
+
+#[inline]
+fn is_compile_time_hook(name: &ClauseName, terms: &Vec<Box<Term>>) -> Option<CompileTimeHook> {
+    if name.as_str() == ":-" {
+        if let Some(ref term) = terms.first() {
+            if let &Term::Clause(_, ref name, ref terms, _) = term.as_ref() {
+                return as_compile_time_hook(name.as_str(), terms.len(), terms);
+            }
+        }
+    }
+
+    as_compile_time_hook(name.as_str(), terms.len(), terms)
+}
+
+type CompileTimeHookCompileInfo = (CompileTimeHook, PredicateClause, VecDeque<TopLevel>);
+
+fn setup_op_decl(mut terms: Vec<Box<Term>>) -> Result<OpDecl, ParserError>
+{
+    let name = match *terms.pop().unwrap() {
+        Term::Constant(_, Constant::Atom(name, _)) => name,
+        _ => return Err(ParserError::InconsistentEntry)
+    };
+
+    let spec = match *terms.pop().unwrap() {
+        Term::Constant(_, Constant::Atom(name, _)) => name,
+        _ => return Err(ParserError::InconsistentEntry)
+    };
+
+    let prec = match *terms.pop().unwrap() {
+        Term::Constant(_, Constant::Number(Number::Integer(bi))) =>
+            match bi.to_usize() {
+                Some(n) if n <= 1200 => n,
+                _ => return Err(ParserError::InconsistentEntry)
+            },
+        _ => return Err(ParserError::InconsistentEntry)
+    };
+
+    match spec.as_str() {
+        "xfx" => Ok(OpDecl(prec, XFX, name)),
+        "xfy" => Ok(OpDecl(prec, XFY, name)),
+        "yfx" => Ok(OpDecl(prec, YFX, name)),
+        "fx"  => Ok(OpDecl(prec, FX, name)),
+        "fy"  => Ok(OpDecl(prec, FY, name)),
+        "xf"  => Ok(OpDecl(prec, XF, name)),
+        "yf"  => Ok(OpDecl(prec, YF, name)),
+        _     => Err(ParserError::InconsistentEntry)
+    }
+}
+
+fn setup_predicate_indicator(mut term: Term) -> Result<PredicateKey, ParserError>
+{
+    match term {
+        Term::Clause(_, ref name, ref mut terms, Some(_))
+            if name.as_str() == "/" && terms.len() == 2 => {
+                let arity = *terms.pop().unwrap();
+                let name  = *terms.pop().unwrap();
+
+                let arity = arity.to_constant().and_then(|c| c.to_integer())
+                    .and_then(|n| if !n.is_negative() { n.to_usize() } else { None })
+                    .ok_or(ParserError::InvalidModuleExport)?;
+
+                let name = name.to_constant().and_then(|c| c.to_atom())
+                    .ok_or(ParserError::InvalidModuleExport)?;
+
+                Ok((name, arity))
+            },
+        _ => Err(ParserError::InvalidModuleExport)
+    }
+}
+
+fn setup_module_decl(mut terms: Vec<Box<Term>>) -> Result<ModuleDecl, ParserError>
+{
+    let mut export_list = *terms.pop().unwrap();
+    let name = terms.pop().unwrap().to_constant().and_then(|c| c.to_atom())
+        .ok_or(ParserError::InvalidModuleDecl)?;
+
+    let mut exports = Vec::new();
+
+    while let Term::Cons(_, t1, t2) = export_list {
+        exports.push(setup_predicate_indicator(*t1)?);
+        export_list = *t2;
+    }
+
+    if export_list.to_constant() != Some(Constant::EmptyList) {
+        Err(ParserError::InvalidModuleDecl)
+    } else {
+        Ok(ModuleDecl { name, exports })
+    }
+}
+
+fn setup_use_module_decl(mut terms: Vec<Box<Term>>) -> Result<ClauseName, ParserError>
+{
+    match *terms.pop().unwrap() {
+        Term::Clause(_, ref name, ref mut terms, None)
+            if name.as_str() == "library" && terms.len() == 1 => {
+                terms.pop().unwrap().to_constant()
+                    .and_then(|c| c.to_atom())
+                    .ok_or(ParserError::InvalidUseModuleDecl)
+            },
+        _ => Err(ParserError::InvalidUseModuleDecl)
+    }
+}
+
+type UseModuleExport = (ClauseName, Vec<PredicateKey>);
+
+fn setup_qualified_import(mut terms: Vec<Box<Term>>) -> Result<UseModuleExport, ParserError>
+{
+    let mut export_list = *terms.pop().unwrap();
+    let name = match *terms.pop().unwrap() {
+        Term::Clause(_, ref name, ref mut terms, None)
+            if name.as_str() == "library" && terms.len() == 1 => {
+                terms.pop().unwrap().to_constant()
+                    .and_then(|c| c.to_atom())
+                    .ok_or(ParserError::InvalidUseModuleDecl)
+            },
+        _ => Err(ParserError::InvalidUseModuleDecl)
+    }?;
+
+    let mut exports = Vec::new();
+
+    while let Term::Cons(_, t1, t2) = export_list {
+        exports.push(setup_predicate_indicator(*t1)?);
+        export_list = *t2;
+    }
+
+    if export_list.to_constant() != Some(Constant::EmptyList) {
+        Err(ParserError::InvalidModuleDecl)
+    } else {
+        Ok((name, exports))
+    }
+}
+
+fn setup_declaration(term: Term) -> Result<Declaration, ParserError>
+{
+    match term {
+        Term::Clause(_, name, mut terms, _) =>
+            if name.as_str() == "op" && terms.len() == 3 {
+                Ok(Declaration::Op(setup_op_decl(terms)?))
+            } else if name.as_str() == "module" && terms.len() == 2 {
+                Ok(Declaration::Module(setup_module_decl(terms)?))
+            } else if name.as_str() == "use_module" && terms.len() == 1 {
+                Ok(Declaration::UseModule(setup_use_module_decl(terms)?))
+            } else if name.as_str() == "use_module" && terms.len() == 2 {
+                let (name, exports) = setup_qualified_import(terms)?;
+                Ok(Declaration::UseQualifiedModule(name, exports))
+            } else if name.as_str() == "non_counted_backtracking" && terms.len() == 1 {
+                let (name, arity) = setup_predicate_indicator(*terms.pop().unwrap())?;
+                Ok(Declaration::NonCountedBacktracking(name, arity))
+            } else if name.as_str() == "dynamic" && terms.len() == 1 {
+                let (name, arity) = setup_predicate_indicator(*terms.pop().unwrap())?;
+                Ok(Declaration::Dynamic(name, arity))
+            } else {
+                Err(ParserError::InconsistentEntry)
+            },
+        _ => return Err(ParserError::InconsistentEntry)
+    }
+}
+
+fn is_consistent(tl: &TopLevel, clauses: &Vec<PredicateClause>) -> bool
+{
+    match clauses.first() {
+        Some(ref cl) => tl.name() == cl.name() && tl.arity() == cl.arity(),
+        None => true
+    }
+}
+
+fn deque_to_packet(head: TopLevel, deque: VecDeque<TopLevel>) -> TopLevelPacket
+{
+    match head {
+        TopLevel::Query(query) => TopLevelPacket::Query(query, deque),
+        tl => TopLevelPacket::Decl(tl, deque)
+    }
+}
+
+fn merge_clauses(tls: &mut VecDeque<TopLevel>) -> Result<TopLevel, ParserError>
+{
+    let mut clauses: Vec<PredicateClause> = vec![];
+
+    while let Some(tl) = tls.pop_front() {
+        match tl {
+            TopLevel::Query(_) if clauses.is_empty() && tls.is_empty() =>
+                return Ok(tl),
+            TopLevel::Declaration(_) if clauses.is_empty() =>
+                return Ok(tl),
+            TopLevel::Query(_) =>
+                return Err(ParserError::InconsistentEntry),
+            TopLevel::Fact(_) if is_consistent(&tl, &clauses) =>
+                if let TopLevel::Fact(fact) = tl {
+                    let clause = PredicateClause::Fact(fact);
+                    clauses.push(clause);
+                },
+            TopLevel::Rule(_) if is_consistent(&tl, &clauses) =>
+                if let TopLevel::Rule(rule) = tl {
+                    let clause = PredicateClause::Rule(rule);
+                    clauses.push(clause);
+                },
+            TopLevel::Predicate(_) if is_consistent(&tl, &clauses) =>
+                if let TopLevel::Predicate(pred) = tl {
+                    clauses.extend(pred.clauses().into_iter())
+                },
+            _ => {
+                tls.push_front(tl);
+                break;
+            }
+        }
+    }
+
+    if clauses.is_empty() {
+        Err(ParserError::InconsistentEntry)
+    } else {
+        Ok(TopLevel::Predicate(Predicate(clauses)))
+    }
+}
+
+fn append_preds(preds: &mut Vec<PredicateClause>) -> Predicate {
+    Predicate(mem::replace(preds, vec![]))
+}
+
+fn mark_cut_variables_as(terms: &mut Vec<Term>, name: ClauseName) {
+    for term in terms.iter_mut() {
+        match term {
+            &mut Term::Constant(_, Constant::Atom(ref mut var, _)) if var.as_str() == "!" =>
+                *var = name.clone(),
+            _ => {}
+        }
+    }
+}
+
+fn mark_cut_variable(term: &mut Term) -> bool {
+    let cut_var_found = match term {
+        &mut Term::Constant(_, Constant::Atom(ref var, _)) if var.as_str() == "!" => true,
+        _ => false
+    };
+
+    if cut_var_found {
+        *term = Term::Var(Cell::default(), rc_atom!("!"));
+        true
+    } else {
+        false
+    }
+}
+
+fn mark_cut_variables(terms: &mut Vec<Term>) -> bool {
+    let mut found_cut_var = false;
+
+    for item in terms.iter_mut() {
+        found_cut_var = mark_cut_variable(item) || found_cut_var;
+    }
+
+    found_cut_var
+}
+
+fn flatten_hook(mut term: Term) -> Term {
+    if let &mut Term::Clause(_, ref mut name, ref mut terms, _) = &mut term {
+        match (name.as_str(), terms.len()) {
+            (":-", 2) => {
+                let inner_term = match terms.first_mut().map(|term| term.borrow_mut()) {
+                    Some(&mut Term::Clause(_, ref name, ref mut inner_terms, _)) =>
+                        if name.as_str() == ":" && inner_terms.len() == 2 {
+                            Some(*inner_terms.pop().unwrap())
+                        } else {
+                            None
+                        },
+                    _ => None
+                };
+
+                if let Some(mut inner_term) = inner_term {
+                    mem::swap(&mut terms[0], &mut Box::new(inner_term));
+                }
+            },
+            (":", 2) => return *terms.pop().unwrap(),
+            _ => {}
+        }
+    }
+
+    term
+}
+
+pub enum TopLevelPacket {
+    Query(Vec<QueryTerm>, VecDeque<TopLevel>),
+    Decl(TopLevel, VecDeque<TopLevel>)
+}
+
+struct RelationWorker {
+    dynamic_clauses: Vec<(Term, Term)>, // Head, Body.
+    queue: VecDeque<VecDeque<Term>>,
+}
+
+impl RelationWorker {
+    fn new() -> Self {
+        RelationWorker { dynamic_clauses: vec![],
+                         queue: VecDeque::new() }
+    }
+
+    fn setup_fact(&mut self, term: Term) -> Result<Term, ParserError>
+    {
+        match term {
+            Term::Clause(..) | Term::Constant(_, Constant::Atom(..)) => {
+                let tail = Term::Constant(Cell::default(),
+                                          Constant::Atom(clause_name!("true"), None));
+
+                self.dynamic_clauses.push((term.clone(), tail));
+                Ok(term)
+            },
+            _ =>
+                Err(ParserError::InadmissibleFact)
+        }
+    }
+
+    fn compute_head(&self, term: &Term) -> Vec<Term>
+    {
+        let mut vars = HashSet::new();
+
+        for term in post_order_iter(term) {
+            if let TermRef::Var(_, _, v) = term {
+                vars.insert(v.clone());
+            }
+        }
+
+        vars.insert(rc_atom!("!"));
+        vars.into_iter()
+            .map(|v| Term::Var(Cell::default(), v))
+            .collect()
+    }
+
+    fn fabricate_rule_body(&self, vars: &Vec<Term>, body_term: Term) -> Term
+    {
+        let vars_of_head = vars.iter().cloned().map(Box::new).collect();
+        let head_term = Term::Clause(Cell::default(), clause_name!(""), vars_of_head, None);
+
+        let rule = vec![Box::new(head_term), Box::new(body_term)];
+        let turnstile = clause_name!(":-");
+
+        Term::Clause(Cell::default(), turnstile, rule, None)
+    }
+
+    // the terms form the body of the rule. We create a head, by
+    // gathering variables from the body of terms and recording them
+    // in the head clause.
+    fn fabricate_rule(&self, body_term: Term) -> (JumpStub, VecDeque<Term>)
+    {
+        // collect the vars of body_term into a head, return the num_vars
+        // (the arity) as well.
+        let vars = self.compute_head(&body_term);
+        let rule = self.fabricate_rule_body(&vars, body_term);
+
+        (vars, VecDeque::from(vec![rule]))
+    }
+
+    fn fabricate_disjunct(&self, body_term: Term) -> (JumpStub, VecDeque<Term>)
+    {
+        let vars = self.compute_head(&body_term);
+        let clauses: Vec<_> = unfold_by_str(body_term, ";").into_iter()
+            .map(|term| {
+                let mut subterms = unfold_by_str(term, ",");
+                mark_cut_variables(&mut subterms);
+
+                let term = subterms.pop().unwrap();
+                fold_by_str(subterms.into_iter(), term, clause_name!(","))
+            }).collect();
+
+        let results = clauses.into_iter()
+            .map(|clause| self.fabricate_rule_body(&vars, clause))
+            .collect();
+
+        (vars, results)
+    }
+
+    fn fabricate_if_then(&self, prec: Term, conq: Term) -> (JumpStub, VecDeque<Term>)
+    {
+        let mut prec_seq = unfold_by_str(prec, ",");
+        let comma_sym    = clause_name!(",");
+        let cut_sym      = atom!("!");
+
+        prec_seq.push(Term::Constant(Cell::default(), cut_sym));
+
+        mark_cut_variables_as(&mut prec_seq, clause_name!("blocked_!"));
+
+        let mut conq_seq = unfold_by_str(conq, ",");
+
+        mark_cut_variables(&mut conq_seq);
+        prec_seq.extend(conq_seq.into_iter());
+
+        let back_term  = Box::new(prec_seq.pop().unwrap());
+        let front_term = Box::new(prec_seq.pop().unwrap());
+
+        let body_term  = Term::Clause(Cell::default(), comma_sym.clone(),
+                                      vec![front_term, back_term], None);
+
+        self.fabricate_rule(fold_by_str(prec_seq.into_iter(), body_term, comma_sym))
+    }
+
+    fn to_query_term(&mut self, indices: &mut CompositeIndices, term: Term) -> Result<QueryTerm, ParserError>
+    {
+        match term {
+            Term::Constant(_, Constant::Atom(name, fixity)) =>
+                if name.as_str() == "!" || name.as_str() == "blocked_!" {
+                    Ok(QueryTerm::BlockedCut)
+                } else {
+                    let ct = indices.get_clause_type(name, 0, fixity);
+                    Ok(QueryTerm::Clause(Cell::default(), ct, vec![], false))
+                },
+            Term::Var(_, ref v) if v.as_str() == "!" =>
+                Ok(QueryTerm::UnblockedCut(Cell::default())),
+            Term::Clause(r, name, mut terms, fixity) =>
+                match (name.as_str(), terms.len()) {
+                    (";", 2) => {
+                        let term = Term::Clause(r, name.clone(), terms, fixity);
+                        let (stub, clauses) = self.fabricate_disjunct(term);
+
+                        self.queue.push_back(clauses);
+                        Ok(QueryTerm::Jump(stub))
+                    },
+                    ("->", 2) => {
+                        let conq = *terms.pop().unwrap();
+                        let prec = *terms.pop().unwrap();
+
+                        let (stub, clauses) = self.fabricate_if_then(prec, conq);
+
+                        self.queue.push_back(clauses);
+                        Ok(QueryTerm::Jump(stub))
+                    },
+                    ("$get_level", 1) =>
+                        if let Term::Var(_, ref var) = *terms[0] {
+                            Ok(QueryTerm::GetLevelAndUnify(Cell::default(), var.clone()))
+                        } else {
+                            Err(ParserError::InadmissibleQueryTerm)
+                        },
+                    ("partial_string", 2) => {
+                        if let Term::Constant(_, Constant::String(_)) = *terms[0].clone() {
+                            if let Term::Var(..) = *terms[1].clone() {
+                                let ct = ClauseType::BuiltIn(BuiltInClauseType::PartialString);
+                                return Ok(QueryTerm::Clause(Cell::default(), ct, terms, false));
+                            }
+                        }
+
+                        Err(ParserError::InadmissibleQueryTerm)
+                    },
+                    _ => {
+                        let ct = indices.get_clause_type(name, terms.len(), fixity);
+                        Ok(QueryTerm::Clause(Cell::default(), ct, terms, false))
+                    }
+                },
+            Term::Var(..) =>
+                Ok(QueryTerm::Clause(Cell::default(), ClauseType::CallN, vec![Box::new(term)], false)),
+            _ => Err(ParserError::InadmissibleQueryTerm)
+        }
+    }
+
+    // never blocks cuts in the consequent.
+    fn prepend_if_then(&self, prec: Term, conq: Term, queue: &mut VecDeque<Box<Term>>,
+                       blocks_cuts: bool)
+    {
+        let cut_symb = atom!("blocked_!");
+        let mut terms_seq = unfold_by_str(prec, ",");
+
+        terms_seq.push(Term::Constant(Cell::default(), cut_symb));
+
+        let mut conq_seq = unfold_by_str(conq, ",");
+
+        if !blocks_cuts {
+            for item in conq_seq.iter_mut() {
+                mark_cut_variable(item);
+            }
+        }
+
+        terms_seq.append(&mut conq_seq);
+
+        while let Some(term) = terms_seq.pop() {
+            queue.push_front(Box::new(term));
+        }
+    }
+
+    fn pre_query_term(&mut self, indices: &mut CompositeIndices, term: Term) -> Result<QueryTerm, ParserError>
+    {
+        match term {
+            Term::Clause(r, name, mut subterms, fixity) =>
+                if subterms.len() == 1 && name.as_str() == "$call_with_default_policy" {
+                    self.to_query_term(indices, *subterms.pop().unwrap())
+                        .map(|mut query_term| {
+                            query_term.set_default_caller();
+                            query_term
+                        })
+                } else {
+                    self.to_query_term(indices, Term::Clause(r, name, subterms, fixity))
+                },
+            _ => self.to_query_term(indices, term)
+        }
+    }
+
+    fn setup_query(&mut self, indices: &mut CompositeIndices, terms: Vec<Box<Term>>, blocks_cuts: bool)
+                   -> Result<Vec<QueryTerm>, ParserError>
+    {
+        let mut query_terms = vec![];
+        let mut work_queue  = VecDeque::from(terms);
+
+        while let Some(term) = work_queue.pop_front() {
+            let mut term = *term;
+
+            // a (->) clause makes up the entire query. That's what the test confirms.
+            if query_terms.is_empty() && work_queue.is_empty() {
+                // check for ->, inline it if found.
+                if let &mut Term::Clause(_, ref name, ref mut subterms, _) = &mut term {
+                    if name.as_str() == "->" && subterms.len() == 2 {
+                        let conq = *subterms.pop().unwrap();
+                        let prec = *subterms.pop().unwrap();
+
+                        self.prepend_if_then(prec, conq, &mut work_queue, blocks_cuts);
+                        continue;
+                    }
+                }
+            }
+
+            for mut subterm in unfold_by_str(term, ",") {
+                if !blocks_cuts {
+                    mark_cut_variable(&mut subterm);
+                }
+
+                query_terms.push(self.pre_query_term(indices, subterm)?);
+            }
+        }
+
+        Ok(query_terms)
+    }
+
+    fn setup_hook(&mut self, hook: CompileTimeHook, indices: &mut CompositeIndices, term: Term)
+                  -> Result<CompileTimeHookCompileInfo, ParserError>
+    {
+        match flatten_hook(term) {
+            Term::Clause(r, name, terms, _) =>
+                if name == hook.name() && terms.len() == hook.arity() {
+                    let term = self.setup_fact(Term::Clause(r, name, terms, None))?;
+                    Ok((hook, PredicateClause::Fact(term), VecDeque::from(vec![])))
+                } else if name.as_str() == ":-" && terms.len() == 2 {
+                    let rule = self.setup_rule(indices, terms, true)?;
+                    let results_queue = self.parse_queue(indices)?;
+
+                    Ok((hook, PredicateClause::Rule(rule), results_queue))
+                } else {
+                    Err(ParserError::InvalidHook)
+                },
+            _ => Err(ParserError::InvalidHook)
+        }
+    }
+
+    fn setup_rule(&mut self, indices: &mut CompositeIndices, mut terms: Vec<Box<Term>>,
+                  blocks_cuts: bool)
+                  -> Result<Rule, ParserError>
+    {
+        let post_head_terms: Vec<_> = terms.drain(1 ..).collect();
+
+        let head = *terms.first().cloned().unwrap();
+        let tail = *post_head_terms.first().cloned().unwrap();
+
+        self.dynamic_clauses.push((head, tail));
+
+        let mut query_terms = self.setup_query(indices, post_head_terms, blocks_cuts)?;
+        let clauses = query_terms.drain(1 ..).collect();
+        let qt = query_terms.pop().unwrap();
+
+        match *terms.pop().unwrap() {
+            Term::Clause(_, name, terms, _) =>
+                Ok(Rule { head: (name, terms, qt), clauses }),
+            Term::Constant(_, Constant::Atom(name, _)) =>
+                Ok(Rule { head: (name, vec![], qt), clauses }),
+            _ => Err(ParserError::InvalidRuleHead)
+        }
+    }
+
+    fn try_term_to_tl(&mut self, indices: &mut CompositeIndices, term: Term, blocks_cuts: bool)
+                      -> Result<TopLevel, ParserError>
+    {
+        match term {
+            Term::Clause(r, name, mut terms, fixity) =>
+                if let Some(hook) = is_compile_time_hook(&name, &terms) {
+                    let term = Term::Clause(r, name, terms, fixity);
+                    let (hook, clause, queue) = self.setup_hook(hook, indices, term)?;
+
+                    Ok(TopLevel::Declaration(Declaration::Hook(hook, clause, queue)))
+                } else if name.as_str() == "?-" {
+                    Ok(TopLevel::Query(try!(self.setup_query(indices, terms, blocks_cuts))))
+                } else if name.as_str() == ":-" && terms.len() > 1 {
+                    Ok(TopLevel::Rule(try!(self.setup_rule(indices, terms, blocks_cuts))))
+                } else if name.as_str() == ":-" && terms.len() == 1 {
+                    let term = *terms.pop().unwrap();
+                    Ok(TopLevel::Declaration(try!(setup_declaration(term))))
+                } else {
+                    let term = Term::Clause(r, name, terms, fixity);
+                    Ok(TopLevel::Fact(try!(self.setup_fact(term))))
+                },
+            term => Ok(TopLevel::Fact(try!(self.setup_fact(term))))
+        }
+    }
+
+    fn try_terms_to_tls<I>(&mut self, indices: &mut CompositeIndices, terms: I, blocks_cuts: bool)
+                           -> Result<VecDeque<TopLevel>, ParserError>
+        where I: IntoIterator<Item=Term>
+    {
+        let mut results = VecDeque::new();
+
+        for term in terms.into_iter() {
+            results.push_back(self.try_term_to_tl(indices, term, blocks_cuts)?);
+        }
+
+        Ok(results)
+    }
+
+    fn parse_queue(&mut self, indices: &mut CompositeIndices) -> Result<VecDeque<TopLevel>, ParserError>
+    {
+        let mut queue = VecDeque::new();
+
+        while let Some(terms) = self.queue.pop_front() {
+            let clauses = merge_clauses(&mut self.try_terms_to_tls(indices, terms, false)?)?;
+            queue.push_back(clauses);
+        }
+
+        Ok(queue)
+    }
+
+    fn absorb(&mut self, other: RelationWorker) {
+        self.queue.extend(other.queue.into_iter());
+        self.dynamic_clauses.extend(other.dynamic_clauses.into_iter());
+    }
+
+    fn expand_queue_contents<'a, R>(&mut self, term_stream: &mut TermStream<'a, R>, op_dir: &OpDir)
+                                    -> Result<(), SessionError>
+        where R: Read
+    {
+        let mut machine_st = MachineState::new();
+        let mut new_queue  = VecDeque::new();
+
+        while let Some(terms) = self.queue.pop_front() {
+            let mut new_terms = VecDeque::new();
+
+            for term in terms {
+                new_terms.push_back(term_stream.run_goal_expanders(&mut machine_st, &op_dir, term)?);
+            }
+
+            new_queue.push_back(new_terms);
+        }
+
+        Ok(self.queue = new_queue)
+    }
+}
+
+fn term_to_toplevel<'a, R>(term_stream: &mut TermStream<'a, R>, code_dir: &mut CodeDir, term: Term)
+                           -> Result<(TopLevel, RelationWorker), ParserError>
+    where R: Read
+{
+    let mut rel_worker = RelationWorker::new();
+    let mut indices = composite_indices!(false, term_stream.indices, code_dir);
+
+    let tl = rel_worker.try_term_to_tl(&mut indices, term, true)?;
+    Ok((tl, rel_worker))
+}
+
+pub
+fn string_to_toplevel<R: Read>(src: R, buffer: String, wam: &mut Machine)
+                               -> Result<TopLevelPacket, SessionError>
+{
+    let mut term_stream = TermStream::new(src, wam.indices.atom_tbl(),
+                                          wam.machine_flags(), &mut wam.indices,
+                                          &mut wam.policies, &mut wam.code_repo);
+
+    term_stream.add_to_top(buffer.as_str());
+
+    let term = term_stream.read_term(&OpDir::new())?;
+    let mut code_dir = CodeDir::new();
+
+    let (tl, mut rel_worker) = term_to_toplevel(&mut term_stream, &mut code_dir, term)?;
+
+    rel_worker.expand_queue_contents(&mut term_stream, &OpDir::new())?;
+
+    let mut indices = composite_indices!(false, term_stream.indices, &mut code_dir);
+    let queue = rel_worker.parse_queue(&mut indices)?;
+
+    Ok(deque_to_packet(tl, queue))
+}
+
+pub type DynamicClauseMap = HashMap<(ClauseName, usize), Vec<(Term, Term)>>;
+
+pub struct TopLevelBatchWorker<'a, R: Read> {
+    pub(crate) term_stream: TermStream<'a, R>,
+    rel_worker: RelationWorker,
+    pub(crate) results: Vec<(Predicate, VecDeque<TopLevel>)>,
+    pub(crate) dynamic_clause_map: DynamicClauseMap,
+    pub(crate) in_module: bool
+}
+
+impl<'a, R: Read> TopLevelBatchWorker<'a, R> {
+    pub fn new(inner: R, atom_tbl: TabledData<Atom>,
+               flags: MachineFlags, indices: &'a mut IndexStore,
+               policies: &'a mut MachinePolicies, code_repo: &'a mut CodeRepo)
+               -> Self
+    {
+        let term_stream = TermStream::new(inner, atom_tbl, flags,
+                                          indices, policies, code_repo);
+
+        TopLevelBatchWorker { term_stream,
+                              rel_worker: RelationWorker::new(),
+                              results: vec![],
+                              dynamic_clause_map: HashMap::new(),
+                              in_module: false }
+    }
+
+    fn try_term_to_tl(&self, indices: &mut IndexStore, term: Term)
+                      -> Result<(TopLevel, RelationWorker), SessionError>
+    {
+        let mut new_rel_worker = RelationWorker::new();
+        let mut indices = composite_indices!(self.in_module, indices,
+                                             &self.term_stream.indices.code_dir);
+
+        Ok((new_rel_worker.try_term_to_tl(&mut indices, term, true)?, new_rel_worker))
+    }
+
+    fn process_result(&mut self, indices: &mut IndexStore, preds: &mut Vec<PredicateClause>)
+                      -> Result<(), SessionError>
+    {
+        self.rel_worker.expand_queue_contents(&mut self.term_stream, &indices.op_dir)?;
+
+        let mut indices = composite_indices!(self.in_module, indices,
+                                             &mut self.term_stream.indices.code_dir);
+
+        let queue  = self.rel_worker.parse_queue(&mut indices)?;
+        let result = (append_preds(preds), queue);
+
+        let in_situ_code_dir = &mut self.term_stream.indices.in_situ_code_dir;
+
+        self.term_stream.code_repo.add_in_situ_result(&result, in_situ_code_dir,
+                                                      self.term_stream.flags)?;
+
+        Ok(self.results.push(result))
+    }
+
+    fn take_dynamic_clauses(&mut self) {
+        let (name, arity) = match self.rel_worker.dynamic_clauses.first() {
+            Some((head, _)) =>
+                (head.name().unwrap(), head.arity()),
+            None =>
+                return
+        };
+
+        match self.dynamic_clause_map.get_mut(&(name.clone(), arity)) {
+            Some(ref mut entry) => {
+                entry.clear(); // don't treat dynamic predicates as if they're discontiguous.
+                entry.extend(self.rel_worker.dynamic_clauses.drain(0 ..));
+            },
+            _ => {
+                self.rel_worker.dynamic_clauses.clear();
+            }
+        }
+    }
+
+    pub fn consume(&mut self, indices: &mut IndexStore) -> Result<Option<Declaration>, SessionError>
+    {
+        let mut preds = vec![];
+
+        while !self.term_stream.eof()? {
+            let term = self.term_stream.read_term(&indices.op_dir)?;
+            let (tl, new_rel_worker) = self.try_term_to_tl(indices, term)?;
+
+            // if is_consistent is false, preds is non-empty.
+            if !is_consistent(&tl, &preds) {
+                self.process_result(indices, &mut preds)?;
+                self.take_dynamic_clauses();
+            }
+
+            self.rel_worker.absorb(new_rel_worker);
+
+            match tl {
+                TopLevel::Fact(fact) => preds.push(PredicateClause::Fact(fact)),
+                TopLevel::Rule(rule) => preds.push(PredicateClause::Rule(rule)),
+                TopLevel::Predicate(pred) => preds.extend(pred.0),
+                TopLevel::Declaration(decl) => return Ok(Some(decl)),
+                TopLevel::Query(_) => return Err(SessionError::NamelessEntry)
+            }
+        }
+
+        if !preds.is_empty() {
+            self.process_result(indices, &mut preds)?;
+            self.take_dynamic_clauses();
+        }
+
+        Ok(None)
+    }
+}