Merge branch 'develop'

Cargo.lock

@@ -86,7 +86,7 @@ dependencies = [
 
 [[package]]
 name = "prolog_parser"
-version = "0.7.14"
+version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -108,12 +108,12 @@ dependencies = [
 
 [[package]]
 name = "rusty-wam"
-version = "0.7.13"
+version = "0.7.14"
 dependencies = [
  "downcast 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "prolog_parser 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "prolog_parser 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -152,7 +152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cacfcab5eb48250ee7d0c7896b51a2c5eec99c1feea5f32025635f5ae4b00070"
 "checksum num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "630de1ef5cc79d0cdd78b7e33b81f083cbfe90de0f4b2b2f07f905867c70e9fe"
 "checksum ordered-float 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "58d25b6c0e47b20d05226d288ff434940296e7e2f8b877975da32f862152241f"
-"checksum prolog_parser 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "5a7a659f57a1c0e9c375a82cc224462dc1b7c259e645fcc283928ef38947b584"
+"checksum prolog_parser 0.7.15 (registry+https://github.com/rust-lang/crates.io-index)" = "11f378539616d1cd7b8fc006f309b5a4b483b82cbab76e898a9f1c99318b4dfa"
 "checksum redox_syscall 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "ab105df655884ede59d45b7070c8a65002d921461ee813a024558ca16030eea0"
 "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
 "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rusty-wam"
-version = "0.7.13"
+version = "0.7.14"
 authors = ["Mark Thom <markjordanthom@gmail.com>"]
 repository = "https://github.com/mthom/rusty-wam"
 description = "The Warren Abstract Machine in Rust."
@@ -10,7 +10,7 @@ license = "BSD-3-Clause"
 downcast = "0.9.1"
 num = "0.2"
 ordered-float = "0.5.0"
-prolog_parser = "0.7.14"
+prolog_parser = "0.7.15"
 
 [dependencies.termion]
 version = "1.4.0"

src/main.rs

@@ -39,8 +39,7 @@ fn prolog_repl() {
                 wam.clear();
                 continue;
             },
-            Err(e) =>
+            Err(e) => print(&mut wam, EvalSession::from(e))
-                print(&mut wam, EvalSession::from(e))
         };
 
         wam.reset();

src/prolog/compile.rs

@@ -235,6 +235,8 @@ impl ListingCompiler {
                     -> Result<(), SessionError>
     {
         match decl {
+            Declaration::Hook(CompileTimeHook::TermExpansion, clause) =>
+                Ok(wam.add_term_expansion_clause(clause)?),
             Declaration::NonCountedBacktracking(name, arity) =>
                 Ok(self.add_non_counted_bt_flag(name, arity)),
             Declaration::Op(op_decl) =>

src/prolog/copier.rs

@@ -9,7 +9,7 @@ pub(crate) struct RedirectInfo {
     trail: Trail
 }
 
-pub(crate) trait CopierTarget
+pub(crate) trait CopierTarget: IndexMut<usize, Output=HeapCellValue>
 {
     fn source(&self) -> usize;
     fn threshold(&self) -> usize;
@@ -19,7 +19,6 @@ pub(crate) trait CopierTarget
     fn stack(&mut self) -> &mut AndStack;
 
     fn unwind_trail(&mut self, redirect: RedirectInfo)
-      where Self: IndexMut<usize, Output=HeapCellValue>
     {
         for (r, hcv) in redirect.trail {
             match r {
@@ -30,7 +29,6 @@ pub(crate) trait CopierTarget
     }
 
     fn reinstantiate_var(&mut self, ra: Addr, scan: usize, trail: &mut Trail)
-        where Self: IndexMut<usize, Output=HeapCellValue>
     {
         self[scan] = HeapCellValue::Addr(Addr::HeapCell(scan));
 
@@ -49,7 +47,6 @@ pub(crate) trait CopierTarget
     // at L1 to L2. trail is kept to restore the innards of L1 after
     // it's been copied to L2.
     fn duplicate_term_impl(&mut self, addr: Addr) -> RedirectInfo
-      where Self: IndexMut<usize, Output=HeapCellValue>
     {
         let mut trail = Trail::new();
         let mut scan = self.source();
@@ -153,7 +150,6 @@ pub(crate) trait CopierTarget
     }
 
     fn duplicate_term(&mut self, addr: Addr)
-      where Self: IndexMut<usize, Output=HeapCellValue>
     {
         let redirect = self.duplicate_term_impl(addr);
         self.unwind_trail(redirect);

src/prolog/instructions.rs

@@ -238,8 +238,7 @@ pub enum SystemClauseType {
     SetDoubleQuotes,
     SkipMaxList,
     Succeed,
-    UnwindStack,
+    UnwindStack
-    CompileAndRunQuery
 }
 
 impl SystemClauseType {
@@ -277,7 +276,6 @@ impl SystemClauseType {
             &SystemClauseType::SkipMaxList => clause_name!("$skip_max_list"),
             &SystemClauseType::Succeed => clause_name!("$succeed"),
             &SystemClauseType::UnwindStack => clause_name!("$unwind_stack"),
-            &SystemClauseType::CompileAndRunQuery => clause_name!("$compile_and_run_query")
         }
     }
 
@@ -311,7 +309,6 @@ impl SystemClauseType {
             ("$set_double_quotes", 1) => Some(SystemClauseType::SetDoubleQuotes),
             ("$skip_max_list", 4) => Some(SystemClauseType::SkipMaxList),
             ("$unwind_stack", 0) => Some(SystemClauseType::UnwindStack),
-            ("$compile_and_run_query", 1) => Some(SystemClauseType::CompileAndRunQuery),
             _ => None
         }
     }
@@ -337,10 +334,22 @@ pub enum BuiltInClauseType {
     Sort,
 }
 
+#[derive(Clone, Copy)]
+pub enum CompileTimeHook {
+    TermExpansion
+}
+
+impl CompileTimeHook {
+    pub fn name(self) -> ClauseName {
+        clause_name!("term_expansion")
+    }
+}
+
 #[derive(Clone)]
 pub enum ClauseType {
     BuiltIn(BuiltInClauseType),
     CallN,
+    Hook(CompileTimeHook),
     Inlined(InlinedClauseType),
     Named(ClauseName, CodeIndex),
     Op(ClauseName, Fixity, CodeIndex),
@@ -442,6 +451,7 @@ impl ClauseType {
         match self {
             &ClauseType::CallN => clause_name!("call"),
             &ClauseType::BuiltIn(ref built_in) => built_in.name(),
+            &ClauseType::Hook(ref hook) => hook.name(),
             &ClauseType::Inlined(ref inlined) => clause_name!(inlined.name()),
             &ClauseType::Op(ref name, ..) => name.clone(),
             &ClauseType::Named(ref name, ..) => name.clone(),
@@ -840,7 +850,7 @@ impl CodePtr {
         match self {
             &CodePtr::BuiltInClause(_, ref local)
           | &CodePtr::CallN(_, ref local)
-          | &CodePtr::Local(ref local) => local.clone()
+          | &CodePtr::Local(ref local) => local.clone()               
         }
     }
 }
@@ -849,6 +859,7 @@ impl CodePtr {
 pub enum LocalCodePtr {
     DirEntry(usize, ClauseName), // offset, resident module name.
     TopLevel(usize, usize), // chunk_num, offset.
+    UserTermExpansion(usize)
 }
 
 impl LocalCodePtr {
@@ -901,7 +912,8 @@ impl Add<usize> for LocalCodePtr {
     fn add(self, rhs: usize) -> Self::Output {
         match self {
             LocalCodePtr::DirEntry(p, name) => LocalCodePtr::DirEntry(p + rhs, name),
-            LocalCodePtr::TopLevel(cn, p) => LocalCodePtr::TopLevel(cn, p + rhs)
+            LocalCodePtr::TopLevel(cn, p) => LocalCodePtr::TopLevel(cn, p + rhs),
+            LocalCodePtr::UserTermExpansion(p) => LocalCodePtr::UserTermExpansion(p + rhs)
         }
     }
 }
@@ -909,8 +921,9 @@ impl Add<usize> for LocalCodePtr {
 impl AddAssign<usize> for LocalCodePtr {
     fn add_assign(&mut self, rhs: usize) {
         match self {
-            &mut LocalCodePtr::DirEntry(ref mut p, _) |
+            &mut LocalCodePtr::UserTermExpansion(ref mut p)
-            &mut LocalCodePtr::TopLevel(_, ref mut p) => *p += rhs
+          | &mut LocalCodePtr::DirEntry(ref mut p, _)
+          | &mut LocalCodePtr::TopLevel(_, ref mut p) => *p += rhs            
         }
     }
 }
@@ -1160,9 +1173,10 @@ impl SubModuleUser for Module {
     }
 }
 
-pub enum Declaration {
+pub enum Declaration {    
-    NonCountedBacktracking(ClauseName, usize), // name, arity
+    Hook(CompileTimeHook, PredicateClause),    
     Module(ModuleDecl),
+    NonCountedBacktracking(ClauseName, usize), // name, arity
     Op(OpDecl),
     UseModule(ClauseName),
     UseQualifiedModule(ClauseName, Vec<PredicateKey>)
@@ -1177,10 +1191,10 @@ impl Declaration {
 
 pub enum TopLevel {
     Declaration(Declaration),
-    Fact(Term),
+    Fact(Term),    
     Predicate(Predicate),
     Query(Vec<QueryTerm>),
-    Rule(Rule)
+    Rule(Rule),    
 }
 
 impl TopLevel {

src/prolog/lib/builtins.pl

@@ -307,5 +307,3 @@ handle_ball(C, C, R) :- !, '$erase_ball', call(R).
 handle_ball(_, _, _) :- '$unwind_stack'.
 
 throw(Ball) :- '$set_ball'(Ball), '$unwind_stack'.
-
-repl :- read(X), '$compile_and_run_query'(X), repl.

src/prolog/machine/machine_state.rs

@@ -527,7 +527,7 @@ pub(crate) trait CallPolicy: Any {
                 machine_st.fail = !machine_st.is_cyclic_term(addr);
                 return_from_clause!(machine_st.last_call, machine_st)
             },
-            &BuiltInClauseType::Read => {                
+            &BuiltInClauseType::Read => {
                 match machine_st.read(stdin(), &indices.op_dir) {
                     Ok(offset) => {
                         let addr = machine_st[temp_v!(1)].clone();
@@ -642,6 +642,17 @@ pub(crate) trait CallPolicy: Any {
         }
     }
 
+    fn compile_hook(&mut self, machine_st: &mut MachineState, _: &CompileTimeHook) -> CallResult
+    {
+        machine_st.cp = LocalCodePtr::TopLevel(0, 0);
+
+        machine_st.num_of_args = 2;
+        machine_st.b0 = machine_st.b;
+        machine_st.p  = CodePtr::Local(LocalCodePtr::UserTermExpansion(0));
+
+        Ok(())
+    }
+
     fn call_n<'a>(&mut self, machine_st: &mut MachineState, arity: usize,
                   indices: MachineCodeIndices<'a>) //code_dirs: CodeDirs)
                   -> CallResult
@@ -674,7 +685,7 @@ pub(crate) trait CallPolicy: Any {
                         return Err(machine_st.error_form(MachineError::existence_error(h, name, arity),
                                                          stub));
                     },
-                ClauseType::System(_) => {
+                ClauseType::Hook(_) | ClauseType::System(_) => {
                     let name = Addr::Con(Constant::Atom(name, None));
                     let stub = MachineError::functor_stub(clause_name!("call"), arity + 1);
 

src/prolog/machine/machine_state_impl.rs

@@ -196,7 +196,7 @@ impl MachineState {
                         self.fail = true;
                     },
                     (Addr::Lis(a1), Addr::Con(Constant::String(ref mut s)))
-                  | (Addr::Con(Constant::String(ref mut s)), Addr::Lis(a1))
+                        | (Addr::Con(Constant::String(ref mut s)), Addr::Lis(a1))
                         if self.flags.double_quotes.is_chars() => {
                             if let Some(c) = s.head() {
                                 pdl.push(Addr::Con(Constant::String(s.tail())));
@@ -233,7 +233,7 @@ impl MachineState {
                             self.fail = true;
                         },
                     (Addr::Con(Constant::EmptyList), Addr::Con(Constant::String(ref s)))
-                  | (Addr::Con(Constant::String(ref s)), Addr::Con(Constant::EmptyList))
+                        | (Addr::Con(Constant::String(ref s)), Addr::Con(Constant::EmptyList))
                         if self.flags.double_quotes.is_chars() => {
                             if s.is_expandable() && s.is_empty() {
                                 s.set_non_expandable();
@@ -314,7 +314,7 @@ impl MachineState {
             }
         }
     }
-
+    
     fn trail(&mut self, r: Ref) {
         match r {
             Ref::HeapCell(hc) =>
@@ -1457,7 +1457,7 @@ impl MachineState {
         for (v1, v2) in iter {
             match (v1, v2) {
                 (HeapCellValue::Addr(Addr::Lis(_)), HeapCellValue::Addr(Addr::Con(Constant::String(_))))
-              | (HeapCellValue::Addr(Addr::Con(Constant::String(_))), HeapCellValue::Addr(Addr::Lis(_)))
+                    | (HeapCellValue::Addr(Addr::Con(Constant::String(_))), HeapCellValue::Addr(Addr::Lis(_)))
                     if self.flags.double_quotes.is_chars() => {},
                 (HeapCellValue::Addr(Addr::Con(Constant::EmptyList)),
                  HeapCellValue::Addr(Addr::Con(Constant::String(ref s))))
@@ -1577,7 +1577,7 @@ impl MachineState {
                 (HeapCellValue::Addr(Addr::Lis(_)), HeapCellValue::Addr(Addr::Lis(_))) =>
                     continue,
                 (HeapCellValue::Addr(Addr::Lis(_)), HeapCellValue::NamedStr(ar, n, _))
-              | (HeapCellValue::NamedStr(ar, n, _), HeapCellValue::Addr(Addr::Lis(_))) =>
+                    | (HeapCellValue::NamedStr(ar, n, _), HeapCellValue::Addr(Addr::Lis(_))) =>
                     if ar == 2 && n.as_str() == "." {
                         continue;
                     } else if ar < 2 {
@@ -2045,7 +2045,7 @@ impl MachineState {
 
         self.p += 1;
     }
-
+    
     fn handle_call_clause<'a>(&mut self, indices: MachineCodeIndices<'a>,
                               call_policy: &mut Box<CallPolicy>,
                               cut_policy:  &mut Box<CutPolicy>,
@@ -2068,6 +2068,8 @@ impl MachineState {
                 try_or_fail!(self, call_policy.call_builtin(self, ct, indices)),
             &ClauseType::CallN =>
                 try_or_fail!(self, call_policy.call_n(self, arity, indices)),
+            &ClauseType::Hook(ref hook) =>
+                try_or_fail!(self, call_policy.compile_hook(self, hook)),
             &ClauseType::Inlined(ref ct) =>
                 self.execute_inlined(ct),
             &ClauseType::Named(ref name, ref idx) | &ClauseType::Op(ref name, _, ref idx) =>
@@ -2087,8 +2089,8 @@ impl MachineState {
             &ControlInstruction::Allocate(num_cells) =>
                 self.allocate(num_cells),
             &ControlInstruction::CallClause(ref ct, arity, _, lco, use_default_cp) =>
-                self.handle_call_clause(indices, call_policy, cut_policy, ct, arity, lco,
+                self.handle_call_clause(indices, call_policy, cut_policy, 
-                                        use_default_cp),
+                                        ct, arity, lco, use_default_cp),
             &ControlInstruction::Deallocate => self.deallocate(),
             &ControlInstruction::JmpBy(arity, offset, _, lco) => {
                 if !lco {

src/prolog/machine/mod.rs

@@ -1,12 +1,15 @@
 use prolog_parser::ast::*;
 use prolog_parser::tabled_rc::*;
 
-use prolog::instructions::*;
+use prolog::codegen::*;
 use prolog::compile::*;
+use prolog::debray_allocator::*;
 use prolog::heap_print::*;
+use prolog::instructions::*;
 
 mod machine_errors;
 pub(super) mod machine_state;
+pub(super) mod term_expansion;
 
 #[macro_use] mod machine_state_impl;
 mod system_calls;
@@ -53,7 +56,8 @@ pub struct Machine {
     code: Code,
     pub(super) code_dir: Rc<RefCell<CodeDir>>,
     pub(super) op_dir: OpDir,
-//    term_dir: TermDir,
+    term_dir: TermDir,
+    term_expanders: Code,
     pub(super) modules: ModuleDir,
     cached_query: Option<Code>
 }
@@ -80,7 +84,8 @@ impl Index<LocalCodePtr> for Machine {
                     &None => panic!("Out-of-bounds top level index.")
                 }
             },
-            LocalCodePtr::DirEntry(p, _) => &self.code[p]
+            LocalCodePtr::DirEntry(p, _) => &self.code[p],
+            LocalCodePtr::UserTermExpansion(p) => &self.term_expanders[p]
         }
     }
 }
@@ -115,7 +120,7 @@ impl<'a> SubModuleUser for MachineCodeIndices<'a> {
 static LISTS: &str   = include_str!("../lib/lists.pl");
 static CONTROL: &str = include_str!("../lib/control.pl");
 static QUEUES: &str  = include_str!("../lib/queues.pl");
-static NUMBERVARS: &str  = include_str!("../lib/numbervars.pl");
+static NUMVARS: &str = include_str!("../lib/numbervars.pl");
 
 impl Machine {
     pub fn new() -> Self {
@@ -126,7 +131,8 @@ impl Machine {
             code: Code::new(),
             code_dir: Rc::new(RefCell::new(CodeDir::new())),
             op_dir: default_op_dir(),
-            // term_dir: TermDir::new(),
+            term_dir: TermDir::new(),
+            term_expanders: Code::new(),
             modules: HashMap::new(),
             cached_query: None
         };
@@ -138,7 +144,7 @@ impl Machine {
         compile_user_module(&mut wam, LISTS.as_bytes());
         compile_user_module(&mut wam, CONTROL.as_bytes());
         compile_user_module(&mut wam, QUEUES.as_bytes());
-	compile_user_module(&mut wam, NUMBERVARS.as_bytes());
+	compile_user_module(&mut wam, NUMVARS.as_bytes());
 
         wam
     }
@@ -245,8 +251,29 @@ impl Machine {
         }
     }
 
+    #[inline]
+    pub(super)
+    fn add_term_expansion_clause(&mut self, clause: PredicateClause) -> Result<(), ParserError>
+    {
+        let key = (clause_name!("term_expansion"), 2);
+        let preds = self.term_dir.entry(key).or_insert(Predicate(vec![]));
+        
+        preds.0.push(clause);
+                
+        let mut cg = CodeGenerator::<DebrayAllocator>::new(false, self.ms.flags);
+        let code = cg.compile_predicate(&preds.0)?;
+
+        Ok(self.term_expanders = code)
+    }
+    
     fn lookup_instr(&self, p: CodePtr) -> Option<Line> {
         match p {
+            CodePtr::Local(LocalCodePtr::UserTermExpansion(p)) =>
+                if p < self.term_expanders.len() {
+                    Some(self.term_expanders[p].clone())
+                } else {
+                    None
+                },
             CodePtr::Local(LocalCodePtr::TopLevel(_, p)) =>
                 match &self.cached_query {
                     &Some(ref cq) => Some(cq[p].clone()),
@@ -342,6 +369,8 @@ impl Machine {
 
             match self.ms.p {
                 CodePtr::Local(LocalCodePtr::DirEntry(p, _)) if p < self.code.len() => {},
+                CodePtr::Local(LocalCodePtr::UserTermExpansion(p)) if p < self.term_expanders.len() => {},
+                CodePtr::Local(LocalCodePtr::UserTermExpansion(_)) => self.ms.fail = true,
                 CodePtr::Local(_) => break,
                 _ => {}
             };

src/prolog/machine/system_calls.rs

@@ -203,7 +203,7 @@ impl MachineState {
             },
             &SystemClauseType::GetDoubleQuotes => {
                 let a1 = self[temp_v!(1)].clone();
-                
+
                 match self.flags.double_quotes {
                     DoubleQuotes::Chars =>
                         self.unify(a1, Addr::Con(atom!("chars"))),
@@ -435,14 +435,7 @@ impl MachineState {
                 return Err(err);
             },
             &SystemClauseType::Succeed => {},
-            &SystemClauseType::UnwindStack => self.unwind_stack(),
+            &SystemClauseType::UnwindStack => self.unwind_stack()
-            &SystemClauseType::CompileAndRunQuery => {}
-/*              let addr = self[temp_v!(1)].clone();
-                
-                match term_write(&self, addr) {
-                    Err(e)   => machine_error
-                    Ok(term) => 
-            }*/
         };
 
         self.set_p();

src/prolog/machine/term_expansion.rs

@@ -0,0 +1,148 @@
+use prolog_parser::ast::*;
+use prolog_parser::parser::*;
+
+use prolog::heap_iter::*;
+use prolog::instructions::HeapCellValue;
+use prolog::machine::*;
+use prolog::read::*;
+
+use std::cell::Cell;
+use std::io::Read;
+
+pub struct TermStream<R: Read> {
+    stack: Vec<Term>,
+    parser: Parser<R>,
+    in_module: bool
+}
+
+impl<R: Read> TermStream<R> {
+    pub fn new(src: R, atom_tbl: TabledData<Atom>, flags: MachineFlags) -> Self {
+        TermStream {
+            stack: Vec::new(),
+            parser: Parser::new(src, atom_tbl, flags),
+            in_module: false
+        }
+    }
+
+    #[inline]
+    pub fn add_to_top(&mut self, buf: &str) {
+        self.parser.add_to_top(buf);
+    }
+
+    #[inline]
+    pub fn eof(&mut self) -> Result<bool, ParserError> {
+        Ok(self.stack.is_empty() && self.parser.eof()?)
+    }
+
+    #[inline]
+    pub fn empty_tokens(&mut self) {
+        self.parser.reset();
+    }
+
+    fn enqueue_term(&mut self, term: Term) -> Result<(), ParserError> {
+        match term {
+            Term::Cons(_, head, tail) => {
+                let mut terms = vec![*head];
+                let mut tail = *tail;
+
+                while let Term::Cons(_, head, next_tail) = tail {
+                    terms.push(*head);
+                    tail = *next_tail;
+                }
+
+                if let Term::Constant(_, Constant::EmptyList) = tail {
+                    Ok(self.stack.extend(terms.into_iter().rev()))
+                } else {
+                    Err(ParserError::ExpectedTopLevelTerm)
+                }
+            },
+            Term::Clause(..) | Term::Constant(_, Constant::Atom(..)) =>
+                Ok(self.stack.push(term)),
+            _ => Err(ParserError::ExpectedTopLevelTerm)
+        }
+    }
+
+    pub fn read_term(&mut self, wam: &mut Machine, op_dir: &OpDir) -> Result<Term, ParserError>
+    {
+        loop {
+            while let Some(term) = self.stack.pop() {
+                match wam.try_expand_term(&term)? {
+                    Some(term) => self.enqueue_term(term)?,
+                    None => return Ok(term)
+                };
+            }
+
+            let term = self.parser.read_term(composite_op!(self.in_module, &wam.op_dir, op_dir))?;
+            self.stack.push(term);
+        }
+    }
+}
+
+impl Machine {
+    fn try_expand_term(&mut self, term: &Term) -> Result<Option<Term>, ParserError> {
+        let term_h = write_term_to_heap(term, &mut self.ms);
+        let h = self.ms.heap.h;
+
+        self.ms[temp_v!(1)] = Addr::HeapCell(term_h);
+        self.ms.heap.push(HeapCellValue::Addr(Addr::HeapCell(h)));
+        self.ms[temp_v!(2)] = Addr::HeapCell(h);
+
+        let code = vec![call_clause!(ClauseType::Hook(CompileTimeHook::TermExpansion), 2, 0, true)];
+        self.submit_query(code, AllocVarDict::new());
+
+        if self.failed() {
+            self.reset();
+            Ok(None)
+        } else {
+            let term = read_term_from_heap(&self.ms, Addr::HeapCell(h))?;
+            self.reset();            
+            Ok(Some(term))
+        }
+    }
+}
+
+pub fn read_term_from_heap(machine_st: &MachineState, addr: Addr) -> Result<Term, ParserError>
+{
+    let pre_order_iter  = HCPreOrderIterator::new(machine_st, addr);
+    let post_order_iter = HCPostOrderIterator::new(pre_order_iter);
+
+    let mut stack = vec![];
+
+    for value in post_order_iter {
+        match value {
+            HeapCellValue::NamedStr(arity, ref name, fixity)
+                if stack.len() >= arity => {
+                    let stack_len = stack.len();
+                    let subterms: Vec<_> = stack.drain(stack_len - arity ..).collect();
+
+                    stack.push(Box::new(Term::Clause(Cell::default(), name.clone(), subterms,
+                                                     fixity)));
+                },
+            HeapCellValue::Addr(Addr::Con(constant)) =>
+                stack.push(Box::new(Term::Constant(Cell::default(), constant))),
+            HeapCellValue::Addr(Addr::Lis(_))
+                if stack.len() >= 2 => {
+                    let stack_len = stack.len();
+                    let (head, tail) = {
+                        let mut iter  = stack.drain(stack_len - 2 ..);
+                        (iter.next().unwrap(), iter.next().unwrap())
+                    };
+
+                    stack.push(Box::new(Term::Cons(Cell::default(), head, tail)));
+                },
+            HeapCellValue::Addr(Addr::HeapCell(h)) =>
+                stack.push(Box::new(Term::Var(Cell::default(), Rc::new(format!("_{}", h))))),
+            HeapCellValue::Addr(Addr::StackCell(fr, sc)) =>
+                stack.push(Box::new(Term::Var(Cell::default(), Rc::new(format!("_{}_{}", sc, fr))))),
+            _ => return Err(ParserError::IncompleteReduction)
+        }
+    }
+
+    if let Some(term) = stack.pop() {
+        if stack.is_empty() {
+            return Ok(*term);
+        }
+    }
+
+    Err(ParserError::IncompleteReduction)
+}

src/prolog/machine/term_writer.rs

@@ -1,54 +0,0 @@
-use prolog_parser::ast::*;
-
-use prolog::heap_iter::*;
-use prolog::instructions::*;
-use prolog::machine::machine_state::MachineState;
-
-use std::cell::Cell;
-use std::rc::Rc;
-
-pub fn term_write(machine_st: &MachineState, addr: Addr) -> Result<Term, ParserError>
-{
-    let pre_order_iter  = HCPreOrderIterator::new(machine_st, addr);
-    let post_order_iter = HCPostOrderIterator::new(pre_order_iter);
-
-    let mut stack = vec![];
-
-    for value in post_order_iter {
-        match value {
-            HeapCellValue::NamedStr(arity, ref name, fixity)
-                if stack.len() >= arity => {
-                    let stack_len = stack.len();
-                    let subterms: Vec<_> = stack.drain(stack_len - arity ..).collect();
-
-                    stack.push(Box::new(Term::Clause(Cell::default(), name.clone(), subterms,
-                                                     fixity)));
-                },
-            HeapCellValue::Addr(Addr::Con(constant)) =>
-                stack.push(Box::new(Term::Constant(Cell::default(), constant))),
-            HeapCellValue::Addr(Addr::Lis(_))
-                if stack.len() >= 2 => {
-                    let stack_len = stack.len();                    
-                    let mut iter  = stack.drain(stack_len - 2 ..);
-                    
-                    let head = iter.next().unwrap();
-                    let tail = iter.next().unwrap();
-
-                    stack.push(Box::new(Term::Cons(Cell::default(), head, tail)));
-                },
-            HeapCellValue::Addr(Addr::HeapCell(h)) =>
-                stack.push(Box::new(Term::Var(Cell::default(), Rc::new(format!("_{}", h))))),
-            HeapCellValue::Addr(Addr::StackCell(fr, sc)) =>
-                stack.push(Box::new(Term::Var(Cell::default(), Rc::new(format!("_{}_{}", sc, fr))))),
-            _ => return Err(ParserError::IncompleteReduction)
-        }
-    }
-
-    if let Some(term) = stack.pop() {
-        if stack.is_empty() {
-            return Ok(*term);
-        }
-    }
-
-    Err(ParserError::IncompleteReduction)
-}

src/prolog/read.rs

@@ -5,6 +5,7 @@ use prolog::instructions::*;
 use prolog::iterators::*;
 use prolog::machine::*;
 use prolog::machine::machine_state::MachineState;
+use prolog::machine::term_expansion::*;
 
 use std::collections::VecDeque;
 use std::io::{Read, stdin};
@@ -29,7 +30,7 @@ pub enum Input {
     Term(Term)
 }
 
-pub fn read_toplevel(wam: &Machine) -> Result<Input, ParserError> {    
+pub fn read_toplevel(wam: &mut Machine) -> Result<Input, ParserError> {    
     let mut buffer = String::new();
 
     let stdin = stdin();
@@ -43,10 +44,10 @@ pub fn read_toplevel(wam: &Machine) -> Result<Input, ParserError> {
             Ok(Input::Batch)
         },
         _ => {
-            let mut parser = Parser::new(stdin.lock(), wam.atom_tbl(), wam.machine_flags());
+            let mut term_stream = TermStream::new(stdin.lock(), wam.atom_tbl(), wam.machine_flags());
-            parser.add_to_top(buffer.as_str());
+            term_stream.add_to_top(buffer.as_str());
 
-            Ok(Input::Term(parser.read_term(composite_op!(&wam.op_dir))?))
+            Ok(Input::Term(term_stream.read_term(wam, &OpDir::new())?))
         }
     }
 }
@@ -57,7 +58,7 @@ impl MachineState {
         let mut parser = Parser::new(inner, self.atom_tbl.clone(), self.flags);
         let term = parser.read_term(composite_op!(op_dir))?;
 
-        Ok(write_term_to_heap(term, self))
+        Ok(write_term_to_heap(&term, self))
     }
 }
 
@@ -77,13 +78,13 @@ fn modify_head_of_queue(machine_st: &mut MachineState, queue: &mut SubtermDeque,
     }
 }
 
-fn write_term_to_heap(term: Term, machine_st: &mut MachineState) -> usize {
+pub(crate) fn write_term_to_heap(term: &Term, machine_st: &mut MachineState) -> usize {
     let h = machine_st.heap.h;
 
     let mut queue = SubtermDeque::new();
     let mut var_dict = HeapVarDict::new();
 
-    for term in breadth_first_iter(&term, true) {
+    for term in breadth_first_iter(term, true) {
         let h = machine_st.heap.h;
 
         match &term {

src/prolog/toplevel.rs

@@ -1,10 +1,10 @@
 use prolog_parser::ast::*;
-use prolog_parser::parser::*;
 use prolog_parser::tabled_rc::*;
 
 use prolog::instructions::*;
 use prolog::iterators::*;
 use prolog::machine::*;
+use prolog::machine::term_expansion::*;
 use prolog::num::*;
 
 use std::collections::{HashSet, VecDeque};
@@ -68,6 +68,21 @@ impl<'a, 'b : 'a> CompositeIndices<'a, 'b>
     }
 }
 
+#[inline]
+fn is_term_expansion(name: &ClauseName, terms: &Vec<Box<Term>>) -> bool {
+    if name.as_str() == ":-" {
+        if let Some(ref term) = terms.first() {
+            if let &Term::Clause(_, ref name, ref terms, None) = term.as_ref() {
+                return (name.as_str(), terms.len()) == ("term_expansion", 2);
+            }
+        }
+    } else if name.as_str() == "term_expansion" {
+        return terms.len() == 2;
+    }
+
+    false
+}
+
 fn setup_fact(term: Term) -> Result<Term, ParserError>
 {
     match term {
@@ -603,6 +618,24 @@ impl RelationWorker {
         Ok(query_terms)
     }
 
+    fn setup_hook(&mut self, indices: &mut CompositeIndices, term: Term)
+                  -> Result<(CompileTimeHook, PredicateClause), ParserError>
+    {
+        match term {
+            Term::Clause(r, name, terms, _) =>
+                if name.as_str() == "term_expansion" && terms.len() == 2 {
+                    let term = Term::Clause(r, name, terms, None);
+                    Ok((CompileTimeHook::TermExpansion, PredicateClause::Fact(term)))
+                } else if name.as_str() == ":-" {
+                    let rule = self.setup_rule(indices, terms, false)?;
+                    Ok((CompileTimeHook::TermExpansion, PredicateClause::Rule(rule)))
+                } else {
+                    Err(ParserError::InvalidHook)
+                },
+            _ => Err(ParserError::InvalidHook)
+        }
+    }
+
     fn setup_rule(&mut self, indices: &mut CompositeIndices, mut terms: Vec<Box<Term>>, blocks_cuts: bool)
                   -> Result<Rule, ParserError>
     {
@@ -625,7 +658,12 @@ impl RelationWorker {
     {
         match term {
             Term::Clause(r, name, mut terms, fixity) =>
-                if name.as_str() == "?-" {
+                if is_term_expansion(&name, &terms) {
+                    let term = Term::Clause(r, name, terms, fixity);
+                    let (hook, clauses) = self.setup_hook(indices, term)?;
+
+                    Ok(TopLevel::Declaration(Declaration::Hook(hook, clauses)))
+                } else if name.as_str() == "?-" {
                     Ok(TopLevel::Query(try!(self.setup_query(indices, terms, blocks_cuts))))
                 } else if name.as_str() == ":-" && terms.len() > 1 {
                     Ok(TopLevel::Rule(try!(self.setup_rule(indices, terms, blocks_cuts))))
@@ -633,7 +671,8 @@ impl RelationWorker {
                     let term = *terms.pop().unwrap();
                     Ok(TopLevel::Declaration(try!(setup_declaration(term))))
                 } else {
-                    Ok(TopLevel::Fact(try!(setup_fact(Term::Clause(r, name, terms, fixity)))))
+                    let term = Term::Clause(r, name, terms, fixity);
+                    Ok(TopLevel::Fact(try!(setup_fact(term))))
                 },
             term => Ok(TopLevel::Fact(try!(setup_fact(term))))
         }
@@ -659,8 +698,8 @@ impl RelationWorker {
         while let Some(terms) = self.queue.pop_front() {
             let clauses = merge_clauses(&mut self.try_terms_to_tls(indices, terms, false)?)?;
             queue.push_back(clauses);
-        }
 
+        }
         Ok(queue)
     }
 
@@ -669,9 +708,12 @@ impl RelationWorker {
     }
 }
 
-// used to parse queries in test. mostly.
+// used to parse queries in test.
+#[cfg(test)]
 pub fn parse_term<R: Read>(wam: &Machine, buf: R) -> Result<Term, ParserError>
 {
+    use prolog_parser::parser::*;
+
     let mut parser = Parser::new(buf, wam.atom_tbl(), wam.machine_flags());
     parser.read_term(composite_op!(&wam.op_dir))
 }
@@ -683,7 +725,7 @@ fn consume_term<'a>(static_code_dir: Rc<RefCell<CodeDir>>, term: Term,
 {
     let mut rel_worker = RelationWorker::new();
     let mut indices = composite_indices!(false, &mut indices, static_code_dir);
-    
+
     let tl = rel_worker.try_term_to_tl(&mut indices, term, true)?;
     let results = rel_worker.parse_queue(&mut indices)?;
 
@@ -691,7 +733,7 @@ fn consume_term<'a>(static_code_dir: Rc<RefCell<CodeDir>>, term: Term,
 }
 
 pub struct TopLevelBatchWorker<R: Read> {
-    parser: Parser<R>,
+    term_stream: TermStream<R>,
     rel_worker: RelationWorker,
     static_code_dir: Rc<RefCell<CodeDir>>,
     pub results: Vec<(Predicate, VecDeque<TopLevel>)>,
@@ -703,19 +745,13 @@ impl<R: Read> TopLevelBatchWorker<R> {
                static_code_dir: Rc<RefCell<CodeDir>>)
                -> Self
     {
-        TopLevelBatchWorker { parser: Parser::new(inner, atom_tbl, flags),
+        TopLevelBatchWorker { term_stream: TermStream::new(inner, atom_tbl, flags),
                               rel_worker: RelationWorker::new(),
                               static_code_dir,
                               results: vec![],
                               in_module: false }
     }
 
-    #[inline]
-    fn read_term(&mut self, static_op_dir: &OpDir, op_dir: &OpDir) -> Result<Term, ParserError> {
-        let composite_op = composite_op!(self.in_module, op_dir, static_op_dir);
-        self.parser.read_term(composite_op)
-    }
-
     pub
     fn consume<'a, 'b : 'a>(&mut self, wam: &mut Machine, indices: &'a mut MachineCodeIndices<'b>)
                            -> Result<Option<Declaration>, SessionError>
@@ -724,12 +760,12 @@ impl<R: Read> TopLevelBatchWorker<R> {
         let mut indices = composite_indices!(self.in_module, indices,
                                              self.static_code_dir.clone());
 
-        while !self.parser.eof()? {
+        while !self.term_stream.eof()? {
-            self.parser.reset(); // empty the parser stack of token descriptions.
+            self.term_stream.empty_tokens(); // empty the parser stack of token descriptions.
 
             let mut new_rel_worker = RelationWorker::new();
-            let term = self.read_term(&wam.op_dir, &indices.local.op_dir)?;          
+            let term = self.term_stream.read_term(wam, &indices.local.op_dir)?;
-                
+
             let tl = new_rel_worker.try_term_to_tl(&mut indices, term, true)?;
 
             if !is_consistent(&tl, &preds) {  // if is_consistent returns false, preds is non-empty.