add expand_goal, make user:term_expansion and user:goal_expansion work properly within modules

README.md

@@ -141,6 +141,7 @@ The following predicates are built-in to rusty-wam.
 * `compound/1`
 * `copy_term/2`
 * `cyclic_term/1`
+* `expand_goal/2`
 * `expand_term/2`
 * `false/0`
 * `float/1`

src/prolog/compile.rs

@@ -75,9 +75,9 @@ fn set_first_index(code: &mut Code)
     }
 }
 
-fn compile_appendix(code: &mut Code, queue: &VecDeque<TopLevel>, non_counted_bt: bool,
+pub fn compile_appendix(code: &mut Code, queue: &VecDeque<TopLevel>, non_counted_bt: bool,
-                    flags: MachineFlags)
+                        flags: MachineFlags)
-                    -> Result<(), ParserError>
+                        -> Result<(), ParserError>
 {
     for tl in queue.iter() {
         set_first_index(code);

src/prolog/heap_print.rs

@@ -528,7 +528,11 @@ impl<'a, Outputter: HCValueOutputter> HCPrinter<'a, Outputter>
             HeapCellValue::Addr(Addr::Con(c)) =>
                 self.print_constant(c, &op),
             HeapCellValue::Addr(Addr::Lis(_)) =>
-                self.push_list(),
+                if self.ignore_ops {
+                    self.format_struct(2, clause_name!("."))
+                } else {
+                    self.push_list()
+                },
             HeapCellValue::Addr(addr) =>
                 if let Some(offset_str) = self.offset_as_string(addr) {
                     push_space_if_amb!(self, &offset_str, &op, {
@@ -572,9 +576,7 @@ impl<'a, Outputter: HCValueOutputter> HCPrinter<'a, Outputter>
                     TokenOrRedirect::Open =>
                         self.outputter.append("("),
                     TokenOrRedirect::OpenList(delimit) =>
-                        if self.ignore_ops {
+                        if !self.at_cdr(", ") {
-                            self.format_struct(2, clause_name!("."));
-                        } else if !self.at_cdr(", ") {
                             self.outputter.append("[");
                         } else {
                             delimit.set(false);

src/prolog/instructions.rs

@@ -218,6 +218,7 @@ pub struct Module {
 #[derive(Copy, Clone, PartialEq)]
 pub enum SystemClauseType {
     CheckCutPoint,
+    ExpandGoal,
     ExpandTerm,
     GetBValue,
     GetSCCCleaner,
@@ -256,6 +257,7 @@ impl SystemClauseType {
         match self {
             &SystemClauseType::CheckCutPoint => clause_name!("$check_cp"),
             &SystemClauseType::ExpandTerm => clause_name!("$expand_term"),
+            &SystemClauseType::ExpandGoal => clause_name!("$expand_goal"),
             &SystemClauseType::GetBValue => clause_name!("$get_b_value"),
             &SystemClauseType::GetDoubleQuotes => clause_name!("$get_double_quotes"),
             &SystemClauseType::GetSCCCleaner => clause_name!("$get_scc_cleaner"),
@@ -292,6 +294,7 @@ impl SystemClauseType {
         match (name, arity) {
             ("$check_cp", 1) => Some(SystemClauseType::CheckCutPoint),
             ("$expand_term", 2) => Some(SystemClauseType::ExpandTerm),
+            ("$expand_goal", 2) => Some(SystemClauseType::ExpandGoal),
             ("$get_b_value", 1) => Some(SystemClauseType::GetBValue),
             ("$get_double_quotes", 1) => Some(SystemClauseType::GetDoubleQuotes),
             ("$get_scc_cleaner", 1) => Some(SystemClauseType::GetSCCCleaner),
@@ -807,9 +810,10 @@ impl HeapCellValue {
 
 #[derive(Clone, Copy, PartialEq)]
 pub enum IndexPtr {
-    Undefined, Index(usize),
+    Undefined,
-    Module // This is a resolved module call. The module
+    Index(usize),
-    // targeted is in the wrapping CodeIndex, and the name is in the ClauseType.
+    Module /* This is a resolved module call. The module
+        targeted is in the wrapping CodeIndex, and the name is in the ClauseType. */
 }
 
 #[derive(Clone)]
@@ -883,6 +887,7 @@ impl CodePtr {
 #[derive(Copy, Clone, PartialEq)]
 pub enum LocalCodePtr {
     DirEntry(usize), // offset.
+    InSituDirEntry(usize),
     TopLevel(usize, usize), // chunk_num, offset.
     UserGoalExpansion(usize),
     UserTermExpansion(usize)
@@ -937,6 +942,7 @@ impl Add<usize> for LocalCodePtr {
 
     fn add(self, rhs: usize) -> Self::Output {
         match self {
+            LocalCodePtr::InSituDirEntry(p) => LocalCodePtr::InSituDirEntry(p + rhs),
             LocalCodePtr::DirEntry(p) => LocalCodePtr::DirEntry(p + rhs),
             LocalCodePtr::TopLevel(cn, p) => LocalCodePtr::TopLevel(cn, p + rhs),
             LocalCodePtr::UserTermExpansion(p) => LocalCodePtr::UserTermExpansion(p + rhs),
@@ -948,7 +954,8 @@ impl Add<usize> for LocalCodePtr {
 impl AddAssign<usize> for LocalCodePtr {
     fn add_assign(&mut self, rhs: usize) {
         match self {
-            &mut LocalCodePtr::UserGoalExpansion(ref mut p)
+            &mut LocalCodePtr::InSituDirEntry(ref mut p)
+          | &mut LocalCodePtr::UserGoalExpansion(ref mut p)
           | &mut LocalCodePtr::UserTermExpansion(ref mut p)
           | &mut LocalCodePtr::DirEntry(ref mut p)
           | &mut LocalCodePtr::TopLevel(_, ref mut p) => *p += rhs

src/prolog/lib/builtins.pl

@@ -6,59 +6,46 @@
 	(=:=)/2, (-)/1, (>=)/2, (=<)/2, (,)/2, (->)/2, (;)/2, (=..)/2,
 	(==)/2, (\==)/2, (@=<)/2, (@>=)/2, (@<)/2, (@>)/2, (=@=)/2,
 	(\=@=)/2, (:)/2, call_with_inference_limit/3, catch/3,
-	current_prolog_flag/2, expand_term/2, set_prolog_flag/2,
+	current_prolog_flag/2, expand_goal/2, expand_term/2,
-	setup_call_cleanup/3, term_variables/2, throw/1, true/0, false/0,
+	set_prolog_flag/2, setup_call_cleanup/3, term_variables/2,
-	write/1, write_canonical/1, writeq/1, write_term/2]).
+	throw/1, true/0, false/0, write/1, write_canonical/1,
+	writeq/1, write_term/2]).
 
 /* this is an implementation specific declarative operator used to implement call_with_inference_limit/3
    and setup_call_cleanup/3. switches to the default trust_me and retry_me_else. Indexing choice
    instructions are unchanged. */
 :- op(700, fx, non_counted_backtracking).
 
+term_expansion((:- op(Pred, Spec, [Op | OtherOps])), OpResults) :-
+    expand_op_list([Op | OtherOps], Pred, Spec, OpResults).
+
+expand_op_list([], _, _, []).
+expand_op_list([Op | OtherOps], Pred, Spec, [(:- op(Pred, Spec, Op)) | OtherResults]) :-
+    expand_op_list(OtherOps, Pred, Spec, OtherResults).
+
 % arithmetic operators.
 :- op(700, xfx, is).
-:- op(500, yfx, +).
+:- op(500, yfx, [+, -]).
-:- op(500, yfx, -).
 :- op(400, yfx, *).
 :- op(200, xfy, **).
-:- op(500, yfx, /\).
+:- op(500, yfx, [/\, \/, xor]).
-:- op(500, yfx, \/).
+:- op(400, yfx, [div, //, rdiv]).
-:- op(500, yfx, xor).
+:- op(400, yfx, [<<, >>, mod, rem]).
-:- op(400, yfx, div).
-:- op(400, yfx, //).
-:- op(400, yfx, rdiv).
-:- op(400, yfx, <<).
-:- op(400, yfx, >>).
-:- op(400, yfx, mod).
-:- op(400, yfx, rem).
 :- op(200, fy, -).
 
 % arithmetic comparison operators.
-:- op(700, xfx, >).
+:- op(700, xfx, [>, <, =\=, =:=, >=, =<]).
-:- op(700, xfx, <).
-:- op(700, xfx, =\=).
-:- op(700, xfx, =:=).
-:- op(700, xfx, >=).
-:- op(700, xfx, =<).
 
 % conditional operators.
 :- op(1050, xfy, ->).
 :- op(1100, xfy, ;).
 
 % control.
-:- op(700, xfx, =).
+:- op(700, xfx, [=, =..]).
 :- op(900, fy, \+).
-:- op(700, xfx, =..).
 
 % term comparison.
-:- op(700, xfx, ==).
+:- op(700, xfx, [==, \==, @=<, @>=, @<, @>, =@=, \=@=]).
-:- op(700, xfx, \==).
-:- op(700, xfx, @=<).
-:- op(700, xfx, @>=).
-:- op(700, xfx, @<).
-:- op(700, xfx, @>).
-:- op(700, xfx, =@=).
-:- op(700, xfx, \=@=).
 
 % module resolution operator.
 :- op(600, xfy, :).
@@ -237,9 +224,13 @@ write_canonical(Term) :- write_term(Term, [ignore_ops(true), quoted(true)]).
 
 writeq(Term) :- write_term(Term, [quoted(true), numbervars(true)]).
 
+% expand_goal.
+
+expand_goal(Term0, Term) :- '$expand_goal'(Term0, Term), !.
+
 % expand_term.
 
-expand_term(Term0, Term) :- '$expand_term'(Term0, Term).
+expand_term(Term0, Term) :- '$expand_term'(Term0, Term), !.
 
 % term_variables.
 

src/prolog/machine/machine_state.rs

@@ -224,19 +224,57 @@ pub struct MachineState {
     pub(crate) flags: MachineFlags
 }
 
-fn call_at_index(machine_st: &mut MachineState, arity: usize, idx: usize)
+fn call_at_index(machine_st: &mut MachineState, arity: usize, p: usize)
 {
     machine_st.cp.assign_if_local(machine_st.p.clone() + 1);
     machine_st.num_of_args = arity;
     machine_st.b0 = machine_st.b;
-    machine_st.p  = dir_entry!(idx);
+    machine_st.p  = dir_entry!(p);
 }
 
-fn execute_at_index(machine_st: &mut MachineState, arity: usize, idx: usize)
+fn execute_at_index(machine_st: &mut MachineState, arity: usize, p: usize)
 {
     machine_st.num_of_args = arity;
     machine_st.b0 = machine_st.b;
-    machine_st.p  = dir_entry!(idx);
+    machine_st.p  = dir_entry!(p);
+}
+
+fn try_in_situ_lookup(name: ClauseName, arity: usize, indices: &IndexStore)
+                      -> Option<usize>
+{
+    match indices.in_situ_code_dir.get(&(name.clone(), arity)) {
+        Some(p) => Some(*p),
+        None => match indices.code_dir.get(&(name, arity)) {
+            Some(ref idx) => if let &IndexPtr::Index(p) = &idx.0.borrow().0 {
+                Some(p)
+            } else {
+                None
+            },
+            _ => None
+        }
+    }
+}
+
+fn try_in_situ(machine_st: &mut MachineState, name: ClauseName, arity: usize,
+               indices: &IndexStore, last_call: bool)
+               -> CallResult
+{
+    if let Some(p) = try_in_situ_lookup(name.clone(), arity, indices) {
+        if last_call {
+            execute_at_index(machine_st, arity, p);
+        } else {
+            call_at_index(machine_st, arity, p);
+        }
+
+        machine_st.p = in_situ_dir_entry!(p);
+        Ok(())
+    } else {
+        let stub = MachineError::functor_stub(name.clone(), arity);
+        let h = machine_st.heap.h;
+        
+        Err(machine_st.error_form(MachineError::existence_error(h, name, arity),
+                                  stub))
+    }
 }
 
 pub(crate) type CallResult = Result<(), Vec<HeapCellValue>>;
@@ -429,13 +467,8 @@ pub(crate) trait CallPolicy: Any {
                 let err = MachineError::module_resolution_error(h, module_name, name, arity);
                 return Err(machine_st.error_form(err, stub));
             },
-            IndexPtr::Undefined => {
+            IndexPtr::Undefined =>
-                let stub = MachineError::functor_stub(name.clone(), arity);
+                return try_in_situ(machine_st, name, arity, indices, false),
-                let h = machine_st.heap.h;
-
-                return Err(machine_st.error_form(MachineError::existence_error(h, name, arity),
-                                                 stub));
-            },
             IndexPtr::Index(compiled_tl_index) =>
                 call_at_index(machine_st, arity, compiled_tl_index)
         }
@@ -464,13 +497,8 @@ pub(crate) trait CallPolicy: Any {
                 let err = MachineError::module_resolution_error(h, module_name, name, arity);
                 return Err(machine_st.error_form(err, stub));
             },
-            IndexPtr::Undefined => {
+            IndexPtr::Undefined =>
-                let stub = MachineError::functor_stub(name.clone(), arity);
+                return try_in_situ(machine_st, name, arity, indices, true),
-                let h = machine_st.heap.h;
-
-                return Err(machine_st.error_form(MachineError::existence_error(h, name, arity),
-                                                 stub));
-            },
             IndexPtr::Index(compiled_tl_index) =>
                 execute_at_index(machine_st, arity, compiled_tl_index)
         }

src/prolog/machine/mod.rs

@@ -1,7 +1,9 @@
 use prolog_parser::ast::*;
 use prolog_parser::tabled_rc::*;
 
+use prolog::codegen::*;
 use prolog::compile::*;
+use prolog::debray_allocator::*;
 use prolog::heap_print::*;
 use prolog::instructions::*;
 
@@ -14,16 +16,19 @@ mod system_calls;
 
 use prolog::machine::machine_state::*;
 
-use std::collections::HashMap;
+use std::collections::{HashMap, VecDeque};
 use std::mem;
 use std::ops::Index;
 use std::rc::Rc;
 
 static BUILTINS: &str = include_str!("../lib/builtins.pl");
 
+pub type InSituCodeDir = HashMap<PredicateKey, usize>;
+
 pub struct IndexStore {
     pub(super) atom_tbl: TabledData<Atom>,
     pub(super) code_dir: CodeDir,
+    pub(super) in_situ_code_dir: InSituCodeDir,
     pub(super) op_dir: OpDir,
     pub(super) modules: ModuleDir
 }
@@ -58,6 +63,7 @@ impl IndexStore {
         IndexStore {
             atom_tbl: TabledData::new(Rc::new("user".to_string())),
             code_dir: CodeDir::new(),
+            in_situ_code_dir: InSituCodeDir::new(),
             op_dir: default_op_dir(),
             modules: ModuleDir::new(),
         }
@@ -100,11 +106,14 @@ impl IndexStore {
     }
 }
 
+pub type CompiledResult = (Predicate, VecDeque<TopLevel>);
+
 pub struct CodeRepo {
     cached_query: Code,
     pub(super) goal_expanders: Code,
     pub(super) term_expanders: Code,
     pub(super) code: Code,
+    pub(super) in_situ_code: Code,
     pub(super) term_dir: TermDir
 }
 
@@ -116,10 +125,33 @@ impl CodeRepo {
             goal_expanders: Code::new(),
             term_expanders: Code::new(),
             code: Code::new(),
+            in_situ_code: Code::new(),
             term_dir: TermDir::new()
         }
     }
 
+    pub fn add_in_situ_result(&mut self, result: &CompiledResult, in_situ_code_dir: &mut InSituCodeDir,
+                              flags: MachineFlags)
+                              -> Result<(), SessionError>
+    {
+        let (ref decl, ref queue) = result;
+        let (name, arity) = decl.0.first().and_then(|cl| {
+            let arity = cl.arity();
+            cl.name().map(|name| (name, arity))
+        }).ok_or(SessionError::NamelessEntry)?;
+
+        let p   = self.in_situ_code.len();
+        in_situ_code_dir.insert((name, arity), p);
+
+        let mut cg = CodeGenerator::<DebrayAllocator>::new(true, flags);
+        let mut decl_code = cg.compile_predicate(&decl.0)?;
+
+        compile_appendix(&mut decl_code, queue, true, flags)?;
+
+        self.in_situ_code.extend(decl_code.into_iter());
+        Ok(())
+    }
+
     #[inline]
     fn size_of_cached_query(&self) -> usize {
         self.cached_query.len()
@@ -146,6 +178,8 @@ impl CodeRepo {
                 } else {
                     None
                 },
+            &CodePtr::Local(LocalCodePtr::InSituDirEntry(p)) =>
+                Some(RefOrOwned::Borrowed(&self.in_situ_code[p])),
             &CodePtr::Local(LocalCodePtr::DirEntry(p)) =>
                 Some(RefOrOwned::Borrowed(&self.code[p])),
             &CodePtr::BuiltInClause(ref built_in, _) => {
@@ -189,6 +223,7 @@ impl Index<LocalCodePtr> for CodeRepo {
 
     fn index(&self, ptr: LocalCodePtr) -> &Self::Output {
         match ptr {
+            LocalCodePtr::InSituDirEntry(p) => &self.in_situ_code[p],
             LocalCodePtr::TopLevel(_, p) => &self.cached_query[p],
             LocalCodePtr::DirEntry(p) => &self.code[p],
             LocalCodePtr::UserGoalExpansion(p) => &self.goal_expanders[p],
@@ -491,7 +526,8 @@ impl MachineState {
         }
     }
 
-    fn query_stepper(&mut self, indices: &mut IndexStore, policies: &mut MachinePolicies, code_repo: &CodeRepo)
+    fn query_stepper(&mut self, indices: &mut IndexStore, policies: &mut MachinePolicies,
+                     code_repo: &CodeRepo)
     {
         loop {
             self.execute_instr(indices, policies, code_repo);
@@ -501,18 +537,27 @@ impl MachineState {
             }
 
             match self.p {
-                CodePtr::Local(LocalCodePtr::DirEntry(p)) if p < code_repo.code.len() => {},
+                CodePtr::Local(LocalCodePtr::DirEntry(p))
-                CodePtr::Local(LocalCodePtr::UserTermExpansion(p)) if p < code_repo.term_expanders.len() => {},
+                    if p < code_repo.code.len() => {},
-                CodePtr::Local(LocalCodePtr::UserTermExpansion(_)) => self.fail = true,
+                CodePtr::Local(LocalCodePtr::UserTermExpansion(p))
-                CodePtr::Local(LocalCodePtr::UserGoalExpansion(p)) if p < code_repo.goal_expanders.len() => {},
+                    if p < code_repo.term_expanders.len() => {},
-                CodePtr::Local(LocalCodePtr::UserGoalExpansion(_)) => self.fail = true,
+                CodePtr::Local(LocalCodePtr::UserTermExpansion(_)) =>
-                CodePtr::Local(_) => break,
+                    self.fail = true,
+                CodePtr::Local(LocalCodePtr::UserGoalExpansion(p))
+                    if p < code_repo.goal_expanders.len() => {},
+                CodePtr::Local(LocalCodePtr::UserGoalExpansion(_)) =>
+                    self.fail = true,
+                CodePtr::Local(LocalCodePtr::InSituDirEntry(p))
+                    if p < code_repo.in_situ_code.len() => {},
+                CodePtr::Local(_) =>
+                    break,
                 _ => {}
             };
         }
     }
 
-    fn record_var_places(&self, chunk_num: usize, alloc_locs: &AllocVarDict, heap_locs: &mut HeapVarDict)
+    fn record_var_places(&self, chunk_num: usize, alloc_locs: &AllocVarDict,
+                         heap_locs: &mut HeapVarDict)
     {
         for (var, var_data) in alloc_locs {
             match var_data {
@@ -571,4 +616,3 @@ impl MachineState {
         }
     }
 }
-

src/prolog/machine/system_calls.rs

@@ -203,9 +203,13 @@ impl MachineState {
                     _ => self.fail = true
                 };
             },
+            &SystemClauseType::ExpandGoal => {
+                self.p = CodePtr::Local(LocalCodePtr::UserGoalExpansion(0));
+//                return Ok(());
+            },
             &SystemClauseType::ExpandTerm => {
                 self.p = CodePtr::Local(LocalCodePtr::UserTermExpansion(0));
-                return Ok(());
+//                return Ok(());
             },
             &SystemClauseType::GetDoubleQuotes => {
                 let a1 = self[temp_v!(1)].clone();

src/prolog/machine/term_expansion.rs

@@ -57,7 +57,14 @@ pub struct TermStream<'a, R: Read> {
     pub(crate) code_repo: &'a mut CodeRepo,
     parser: Parser<R>,
     in_module: bool,
-    flags: MachineFlags
+    pub(crate) flags: MachineFlags
+}
+
+impl<'a, R: Read> Drop for TermStream<'a, R> {
+    fn drop(&mut self) {
+        self.indices.in_situ_code_dir.clear();
+        self.code_repo.in_situ_code.clear();
+    }
 }
 
 impl<'a, R: Read> TermStream<'a, R> {
@@ -243,6 +250,7 @@ impl MachineState {
 
                 printer.quoted = true;
                 printer.numbervars = true;
+                printer.ignore_ops = true;
 
                 printer.see_all_locs();
 

src/prolog/macros.rs

@@ -200,6 +200,12 @@ macro_rules! dir_entry {
     )
 }
 
+macro_rules! in_situ_dir_entry {
+    ($idx:expr) => (
+        CodePtr::Local(LocalCodePtr::InSituDirEntry($idx))
+    )
+}
+
 macro_rules! set_code_index {
     ($idx:expr, $ip:expr, $mod_name:expr) => {{
         let mut idx = $idx.0.borrow_mut();
@@ -213,6 +219,7 @@ macro_rules! index_store {
     ($atom_tbl:expr, $code_dir:expr, $op_dir:expr, $modules:expr) => (
         IndexStore { atom_tbl: $atom_tbl,
                      code_dir: $code_dir,
+                     in_situ_code_dir: InSituCodeDir::new(),
                      op_dir: $op_dir,                     
                      modules: $modules }
     )

src/prolog/toplevel.rs

@@ -699,8 +699,8 @@ pub
 fn consume_term(term: Term, indices: &mut IndexStore) -> Result<TopLevelPacket, ParserError>
 {
     let mut rel_worker = RelationWorker::new();
-    let mut _code_dir = CodeDir::new();
+    let mut code_dir = CodeDir::new();
-    let mut indices = composite_indices!(false, indices, &mut _code_dir);
+    let mut indices = composite_indices!(false, indices, &mut code_dir);
 
     let tl = rel_worker.try_term_to_tl(&mut indices, term, true)?;
     let results = rel_worker.parse_queue(&mut indices)?;
@@ -749,7 +749,13 @@ impl<'a, R: Read> TopLevelBatchWorker<'a, R> {
             // if is_consistent returns false, preds is non-empty.
             if !is_consistent(&tl, &preds) {
                 let result_queue = self.rel_worker.parse_queue(&mut indices)?;
-                self.results.push((append_preds(&mut preds), result_queue));
+                let result = (append_preds(&mut preds), result_queue);
+                let in_situ_code_dir = &mut self.term_stream.indices.in_situ_code_dir;
+
+                self.term_stream.code_repo.add_in_situ_result(&result,
+                                                              in_situ_code_dir,
+                                                              self.term_stream.flags)?;
+                self.results.push(result);
             }
 
             self.rel_worker.absorb(new_rel_worker);