add preliminary multifile declaration support

src/prolog/forms.rs

@@ -203,6 +203,7 @@ pub enum Declaration {
     Hook(CompileTimeHook, PredicateClause, VecDeque<TopLevel>),
     ModuleInitialization(Vec<QueryTerm>, VecDeque<TopLevel>), // goal
     Module(ModuleDecl),
+    MultiFile(ClauseName, usize),
     NonCountedBacktracking(ClauseName, usize), // name, arity
     Op(OpDecl),
     UseModule(ModuleSource),
@@ -371,6 +372,7 @@ pub struct Module {
     pub module_decl: ModuleDecl,
     pub code_dir: CodeDir,
     pub op_dir: OpDir,
+    pub term_dir: TermDir, // this contains multifile predicates.
     pub term_expansions: (Predicate, VecDeque<TopLevel>),
     pub goal_expansions: (Predicate, VecDeque<TopLevel>),
     pub user_term_expansions: (Predicate, VecDeque<TopLevel>), // term expansions inherited from the user scope.

src/prolog/lib/builtins.pl

@@ -614,8 +614,10 @@ assertz_clause(Head, Body) :-
 	 arg(1, Head, Module),
 	 arg(2, Head, F),
 	 module_assertz_clause(F, Body, Module)
-       ; '$no_such_predicate'(Head) -> call_assertz(Head, Body, Name, Arity)
+       ; '$no_such_predicate'(Head) ->
-       ; '$head_is_dynamic'(Head) -> call_assertz(Head, Body, Name, Arity)
+	 call_assertz(Head, Body, Name, Arity)
+       ; '$head_is_dynamic'(Head) ->
+	 call_assertz(Head, Body, Name, Arity)
        ; throw(error(permission_error(modify, static_procedure, Name/Arity), assertz/1))
        )
     ;  throw(error(type_error(callable, Head), assertz/1))

src/prolog/machine/compile.rs

@@ -374,7 +374,17 @@ fn compile_into_module_impl<R: Read>(
     let mut clause_code_generator = ClauseCodeGenerator::new(module_code.len(), module_name.clone());
 
     clause_code_generator.generate_clause_code(&results.dynamic_clause_map, wam)?;
-    add_module_code(wam, compiler.module.take().unwrap(), module_code, indices);
+
+    let top_level_term_dir = results.top_level_term_dirs.consolidate();
+    
+    add_module_code(
+        wam,
+        compiler.module.take().unwrap(),
+        module_code,
+        indices,
+        top_level_term_dir,
+    );
+    
     clause_code_generator.add_clause_code(wam, results.dynamic_clause_map);
 
     Ok(compiler.drop_expansions(wam.machine_flags(), &mut wam.code_repo))
@@ -386,7 +396,9 @@ pub struct GatherResult {
     toplevel_results: Vec<PredicateCompileQueue>,
     toplevel_indices: IndexStore,
     addition_results: ExpansionAdditionResult,
-    top_level_terms: Vec<(Term, usize, usize)>
+    top_level_terms: Vec<(Term, usize, usize)>,
+    top_level_term_dirs: TermDirQuantum,
+    module_term_dirs: TermDirQuantum,
 }
 
 pub struct ClauseCodeGenerator {
@@ -489,16 +501,29 @@ pub struct ListingCompiler {
     listing_src: ListingSource, // a file? a module?
 }
 
-fn add_toplevel_code(wam: &mut Machine, code: Code, indices: IndexStore) {
+fn add_toplevel_code(
+    wam: &mut Machine,
+    code: Code,
+    indices: IndexStore,
+    term_dir: TermDir,    
+) {
     wam.add_batched_code(code, indices.code_dir);
     wam.add_batched_ops(indices.op_dir);
+
+    wam.code_repo.term_dir.extend(term_dir.into_iter());
 }
 
 #[inline]
-fn add_module_code(wam: &mut Machine, mut module: Module, code: Code, indices: IndexStore)
+fn add_module_code(
-{
+    wam: &mut Machine,
+    mut module: Module,
+    code: Code,
+    indices: IndexStore,
+    term_dir: TermDir,
+) {
     module.code_dir.extend(indices.code_dir);
     module.op_dir.extend(indices.op_dir.into_iter());
+    module.term_dir.extend(term_dir.into_iter());
 
     wam.add_module(module, code);
 }
@@ -508,13 +533,14 @@ fn add_non_module_code(
     dynamic_clause_map: DynamicClauseMap,
     code: Code,
     indices: IndexStore,
+    term_dir: TermDir,
 ) -> Result<(), SessionError> {
     wam.check_toplevel_code(&indices)?;
 
     let mut clause_code_generator = ClauseCodeGenerator::new(code.len(), clause_name!("user"));
     clause_code_generator.generate_clause_code(&dynamic_clause_map, wam)?;
 
-    add_toplevel_code(wam, code, indices);
+    add_toplevel_code(wam, code, indices, term_dir);
     clause_code_generator.add_clause_code(wam, dynamic_clause_map);
 
     Ok(())
@@ -694,6 +720,7 @@ impl ListingCompiler {
             let (name, arity) = decl
                 .predicate_indicator()
                 .ok_or(SessionError::NamelessEntry)?;
+            
             let non_counted_bt = self.non_counted_bt_preds.contains(&(name.clone(), arity));
 
             let p = code.len() + wam.code_repo.code.len() + code_offset;
@@ -797,7 +824,12 @@ impl ListingCompiler {
         flags: MachineFlags,
     ) -> Result<(), SessionError> {
         match decl {
-            Declaration::EndOfFile => Ok(()),
+            Declaration::Dynamic(..) => {
+                Ok(())
+            }
+            Declaration::EndOfFile => {
+                Ok(())
+            }
             Declaration::Hook(hook, clause, queue) => {
                 let key = (hook.name(), hook.arity());
                 let (len, queue_len) =
@@ -812,37 +844,6 @@ impl ListingCompiler {
 
                 result
             }
-            Declaration::NonCountedBacktracking(name, arity) => {
-                Ok(self.add_non_counted_bt_flag(name, arity))
-            }
-            Declaration::Op(op_decl) => {
-                self.submit_op(wam, indices, &op_decl)
-            }
-            Declaration::UseModule(ModuleSource::Library(name)) => {
-                let name = if !wam.indices.modules.contains_key(&name) {
-                    load_library(wam, name, true)?
-                } else {
-                    name
-                };
-
-                self.use_module(name, &mut wam.code_repo, flags, &mut wam.indices, indices)
-            }
-            Declaration::UseQualifiedModule(ModuleSource::Library(name), exports) => {
-                let name = if !wam.indices.modules.contains_key(&name) {
-                    load_library(wam, name, true)?
-                } else {
-                    name
-                };
-
-                self.use_qualified_module(
-                    name,
-                    &mut wam.code_repo,
-                    flags,
-                    &exports,
-                    &mut wam.indices,
-                    indices
-                )
-            },
             Declaration::Module(module_decl) => {
                 if self.module.is_none() {
                     let module_name = module_decl.name.clone();
@@ -861,6 +862,30 @@ impl ListingCompiler {
                     Err(SessionError::from(ParserError::InvalidModuleDecl))
                 }
             }
+	    Declaration::ModuleInitialization(query_terms, queue) => {
+		self.initialization_goals.0.extend(query_terms.into_iter());
+		self.initialization_goals.1.extend(queue.into_iter());
+
+		Ok(())
+	    }
+            Declaration::MultiFile(..) => {
+                Ok(())
+            }
+            Declaration::NonCountedBacktracking(name, arity) => {
+                Ok(self.add_non_counted_bt_flag(name, arity))
+            }
+            Declaration::Op(op_decl) => {
+                self.submit_op(wam, indices, &op_decl)
+            }
+            Declaration::UseModule(ModuleSource::Library(name)) => {
+                let name = if !wam.indices.modules.contains_key(&name) {
+                    load_library(wam, name, true)?
+                } else {
+                    name
+                };
+
+                self.use_module(name, &mut wam.code_repo, flags, &mut wam.indices, indices)
+            }
             Declaration::UseModule(ModuleSource::File(filename)) => {
                 let mut path_buf = self.listing_src.path();
                 path_buf.push(filename.as_str());
@@ -868,6 +893,22 @@ impl ListingCompiler {
                 let name = load_module_from_file(wam, path_buf, true)?;
                 self.use_module(name, &mut wam.code_repo, flags, &mut wam.indices, indices)
             }
+            Declaration::UseQualifiedModule(ModuleSource::Library(name), exports) => {
+                let name = if !wam.indices.modules.contains_key(&name) {
+                    load_library(wam, name, true)?
+                } else {
+                    name
+                };
+
+                self.use_qualified_module(
+                    name,
+                    &mut wam.code_repo,
+                    flags,
+                    &exports,
+                    &mut wam.indices,
+                    indices
+                )
+            }
             Declaration::UseQualifiedModule(ModuleSource::File(filename), exports) => {
                 let mut path_buf = self.listing_src.path();
                 path_buf.push(filename.as_str());
@@ -883,14 +924,55 @@ impl ListingCompiler {
                     indices,
                 )
             }
-	    Declaration::ModuleInitialization(query_terms, queue) => {
+        }
-		self.initialization_goals.0.extend(query_terms.into_iter());
+    }
-		self.initialization_goals.1.extend(queue.into_iter());
 
-		Ok(())
+    fn setup_multifile_decl<R: Read>(
+        &self,
+        name: ClauseName,
+        arity: usize,
+        worker: &mut TopLevelBatchWorker<R>
+    ) -> Result<(), SessionError> {
+        let module_name = name.owning_module();
+
+        let term_dir = match module_name.as_str() {
+            "user" => {
+                &worker.term_stream.wam.code_repo.term_dir
             }
-            Declaration::Dynamic(..) => Ok(()),
+            _ => {
+                if let Some(ref module) = self.module {
+                    &module.term_dir
+                } else {
+                    match worker.term_stream.wam.indices.modules.get(&module_name) {
+                        Some(ref module) => {
+                            &module.term_dir
                         }
+                        None => {
+                            return Err(SessionError::ModuleNotFound);
+                        }
+                    }
+                }
+            }
+        };
+
+        Ok(match term_dir.get(&(name.clone(), arity)) {
+            Some((ref preds, ref queue)) => {
+                let (preds, queue) = (preds.clone(), queue.clone());
+
+                worker.term_dirs.set_old(
+                    (name.clone(), arity),
+                    preds,
+                    queue,
+                );
+            }
+            None => {
+                worker.term_dirs.set_old(
+                    (name.clone(), arity),
+                    Predicate::new(),
+                    VecDeque::new(),                        
+                );
+            }
+        })
     }
 
     fn process_and_commit_decl<R: Read>(
@@ -919,6 +1001,9 @@ impl ListingCompiler {
             &Declaration::Hook(hook, _, ref queue) if !hook.has_module_scope() => {
                 worker.term_stream.incr_expansion_lens(hook, 1, queue.len())
             }
+            &Declaration::MultiFile(ref name, arity) => {
+                self.setup_multifile_decl(name.clone(), arity, worker)?;
+            }
             &Declaration::UseModule(_) | &Declaration::UseQualifiedModule(..) => {
                 update_expansion_lengths = true
             }
@@ -947,6 +1032,8 @@ impl ListingCompiler {
         let mut toplevel_results = vec![];
         let mut toplevel_indices = default_index_store!(atom_tbl.clone());
 
+        let mut top_level_term_dirs = TermDirQuantum::new();
+
         while let Some(decl) = worker.consume(indices)? {
             if decl.is_module_decl() {
                 toplevel_indices.copy_and_swap(indices);
@@ -957,6 +1044,10 @@ impl ListingCompiler {
                 
                 if let Some(ref module) = &self.module {
                     worker.term_stream.set_atom_tbl(module.atom_tbl.clone());
+                    top_level_term_dirs = mem::replace(
+                        &mut worker.term_dirs,
+                        TermDirQuantum::new(),
+                    );
                 }
             } else if decl.is_end_of_file() {
                 break;
@@ -967,6 +1058,13 @@ impl ListingCompiler {
 
         let addition_results = worker.term_stream.rollback_expansion_code()?;        
 
+        let module_term_dirs = if self.module.is_some() {
+            worker.term_dirs
+        } else {
+            top_level_term_dirs = worker.term_dirs;
+            TermDirQuantum::new()
+        };
+        
         Ok(GatherResult {
             worker_results: worker.results,
             dynamic_clause_map: worker.dynamic_clause_map,
@@ -974,6 +1072,8 @@ impl ListingCompiler {
             toplevel_indices,
             addition_results,
             top_level_terms: worker.term_stream.top_level_terms(),
+            top_level_term_dirs,
+            module_term_dirs,
         })
     }
 
@@ -1016,6 +1116,9 @@ fn compile_work_impl(
         }
     }
 
+    let top_level_term_dir = results.top_level_term_dirs.consolidate();
+    let module_term_dir = results.module_term_dirs.consolidate();
+
     let module_code = compiler.generate_code(
         results.worker_results,
         wam,
@@ -1062,24 +1165,25 @@ fn compile_work_impl(
         }
 
         if module.is_impromptu_module {
-            add_module_code(wam, module, module_code, indices);
+            add_module_code(wam, module, module_code, indices, module_term_dir);
 
             let module = wam.indices.take_module(compiler.listing_src.name()).unwrap();
 
             wam.indices.use_module(&mut wam.code_repo, wam.machine_st.flags, &module)?;
             wam.indices.insert_module(module);
         } else {
-            add_module_code(wam, module, module_code, indices);
+            add_module_code(wam, module, module_code, indices, module_term_dir);
         }
 
-        add_toplevel_code(wam, toplvl_code, results.toplevel_indices);
+        add_toplevel_code(wam, toplvl_code, results.toplevel_indices, top_level_term_dir);
         clause_code_generator.add_clause_code(wam, results.dynamic_clause_map);
     } else {
         add_non_module_code(
             wam,
             results.dynamic_clause_map,
             module_code,
-            indices
+            indices,
+            top_level_term_dir,
         )?;
     }
 
@@ -1132,7 +1236,9 @@ pub fn compile_special_form<R: Read>(
     let code = compiler.generate_code(results.worker_results, wam, &mut indices.code_dir, 0)?;
     let p = wam.code_repo.code.len();
 
-    add_toplevel_code(wam, code, indices);
+    let top_level_term_dir = results.top_level_term_dirs.consolidate();
+    
+    add_toplevel_code(wam, code, indices, top_level_term_dir);
 
     Ok(p)
 }

src/prolog/machine/dynamic_database.rs

@@ -270,8 +270,12 @@ impl Machine {
         p: LocalCodePtr,
     ) {
         match trans_type {
-            DynamicTransactionType::Abolish => self.abolish_dynamic_clause(temp_v!(1), temp_v!(2)),
+            DynamicTransactionType::Abolish => {
-            DynamicTransactionType::Assert(place) => self.recompile_dynamic_predicate(place),
+                self.abolish_dynamic_clause(temp_v!(1), temp_v!(2))
+            }
+            DynamicTransactionType::Assert(place) => {
+                self.recompile_dynamic_predicate(place)
+            }
             DynamicTransactionType::ModuleAbolish => {
                 self.abolish_dynamic_clause_in_module(temp_v!(1), temp_v!(2), temp_v!(3))
             }
@@ -281,7 +285,9 @@ impl Machine {
             DynamicTransactionType::ModuleRetract => {
                 self.retract_from_dynamic_predicate_in_module()
             }
-            DynamicTransactionType::Retract => self.retract_from_dynamic_predicate(),
+            DynamicTransactionType::Retract => {
+                self.retract_from_dynamic_predicate()
+            }
         }
 
         self.machine_st.p = CodePtr::Local(p);

src/prolog/machine/modules.rs

@@ -18,8 +18,9 @@ impl Module {
     ) -> Self
     {
         Module {
-            module_decl,
             atom_tbl,
+            module_decl,
+            term_dir: TermDir::new(),
             user_term_expansions: (Predicate::new(), VecDeque::from(vec![])),
             user_goal_expansions: (Predicate::new(), VecDeque::from(vec![])),
             term_expansions: (Predicate::new(), VecDeque::from(vec![])),

src/prolog/machine/toplevel.rs

@@ -377,10 +377,6 @@ fn merge_clauses(tls: &mut VecDeque<TopLevel>) -> Result<TopLevel, ParserError>
     }
 }
 
-fn append_preds(preds: &mut Vec<PredicateClause>) -> Predicate {
-    Predicate(mem::replace(preds, vec![]))
-}
-
 fn mark_cut_variables_as(terms: &mut Vec<Term>, name: ClauseName) {
     for term in terms.iter_mut() {
         match term {
@@ -493,20 +489,6 @@ fn setup_declaration<'a, 'b, 'c, R: Read>(
     match term {
         Term::Clause(_, name, mut terms, _) =>
 	    match (name.as_str(), terms.len()) {
-		("op", 3) =>
-		    Ok(Declaration::Op(setup_op_decl(terms, indices.atom_tbl())?)),
-		("module", 2) =>
-		    Ok(Declaration::Module(setup_module_decl(terms, indices.atom_tbl())?)),
-		("use_module", 1) =>
-		    Ok(Declaration::UseModule(setup_use_module_decl(terms)?)),
-		("use_module", 2) => {
-		    let (name, exports) = setup_qualified_import(terms, indices.atom_tbl())?;
-		    Ok(Declaration::UseQualifiedModule(name, exports))
-		}
-		("non_counted_backtracking", 1) => {
-		    let (name, arity) = setup_predicate_indicator(&mut *terms.pop().unwrap())?;
-		    Ok(Declaration::NonCountedBacktracking(name, arity))
-		}
 		("dynamic", 1) => {
 		    let (name, arity) = setup_predicate_indicator(&mut *terms.pop().unwrap())?;
 		    Ok(Declaration::Dynamic(name, arity))
@@ -518,10 +500,30 @@ fn setup_declaration<'a, 'b, 'c, R: Read>(
 
 		    Ok(Declaration::ModuleInitialization(query_terms, queue))
 		}
-		_ =>
+		("module", 2) =>
+		    Ok(Declaration::Module(setup_module_decl(terms, indices.atom_tbl())?)),
+		("op", 3) =>
+		    Ok(Declaration::Op(setup_op_decl(terms, indices.atom_tbl())?)),
+		("non_counted_backtracking", 1) => {
+		    let (name, arity) = setup_predicate_indicator(&mut *terms.pop().unwrap())?;
+		    Ok(Declaration::NonCountedBacktracking(name, arity))
+		}
+                ("multifile", 1) => {
+                    let (name, arity) = setup_predicate_indicator(&mut *terms.pop().unwrap())?;
+                    Ok(Declaration::MultiFile(name, arity))
+                }
+		("use_module", 1) => {
+		    Ok(Declaration::UseModule(setup_use_module_decl(terms)?))
+                }
+		("use_module", 2) => {
+		    let (name, exports) = setup_qualified_import(terms, indices.atom_tbl())?;
+		    Ok(Declaration::UseQualifiedModule(name, exports))
+		}
+		_ => {
 		    Err(ParserError::InconsistentEntry)
+                }
 	    },
-        _ => return Err(ParserError::InconsistentEntry),
+        _ => Err(ParserError::InconsistentEntry),
     }
 }
 
@@ -943,7 +945,64 @@ impl RelationWorker {
     }
 }
 
-pub type DynamicClauseMap = IndexMap<(ClauseName, usize), Vec<(Term, Term)>>;
+pub type DynamicClause = Vec<(Term, Term)>;
+
+pub type DynamicClauseMap = IndexMap<(ClauseName, usize), DynamicClause>;
+
+pub struct TermDirQuantum {
+    old_term_dir: TermDir,
+    new_term_dir: TermDir,
+}
+
+impl TermDirQuantum {
+    pub fn new() -> Self {
+        Self {
+            old_term_dir: TermDir::new(),
+            new_term_dir: TermDir::new()
+        }
+    }
+
+    #[inline]
+    fn get_old(&self, name: ClauseName, arity: usize) -> Option<&(Predicate, VecDeque<TopLevel>)>
+    {
+        self.old_term_dir.get(&(name, arity))
+    }
+
+    #[inline]
+    fn add_new(
+        &mut self,
+        key: PredicateKey,
+        preds: &mut Vec<PredicateClause>,
+        queue: VecDeque<TopLevel>
+    ) {
+        let preds = Predicate(mem::replace(preds, vec![]));
+        self.new_term_dir.insert(key, (preds, queue));
+    }
+
+    pub fn consolidate(self) -> TermDir {
+        let mut term_dir = self.old_term_dir;
+
+        for (key, (preds, queue)) in self.new_term_dir {
+            let (prev_preds, prev_queue) =
+                term_dir.entry(key).or_insert((Predicate::new(), VecDeque::new()));
+
+            prev_preds.0.extend(preds.0.into_iter());
+            prev_queue.extend(queue.into_iter());
+        }
+
+        term_dir
+    }
+
+    #[inline]
+    pub fn set_old(
+        &mut self,
+        key: PredicateKey,
+        pred: Predicate,
+        queue: VecDeque<TopLevel>
+    ) {
+        self.old_term_dir.insert(key, (pred, queue));
+    }
+}
 
 pub struct TopLevelBatchWorker<'a, R: Read> {
     pub(crate) term_stream: TermStream<'a, R>,
@@ -951,6 +1010,7 @@ pub struct TopLevelBatchWorker<'a, R: Read> {
     pub(crate) results: Vec<(Predicate, VecDeque<TopLevel>)>,
     pub(crate) dynamic_clause_map: DynamicClauseMap,
     pub(crate) in_module: bool,
+    pub(crate) term_dirs: TermDirQuantum
 }
 
 impl<'a, R: Read> TopLevelBatchWorker<'a, R> {
@@ -971,6 +1031,7 @@ impl<'a, R: Read> TopLevelBatchWorker<'a, R> {
             results: vec![],
             dynamic_clause_map: IndexMap::new(),
             in_module: false,
+            term_dirs: TermDirQuantum::new(),
         }
     }
 
@@ -1006,8 +1067,22 @@ impl<'a, R: Read> TopLevelBatchWorker<'a, R> {
             if self.in_module { None } else { Some(IndexSource::TermStream) }
         );
 
+        let (name, arity) = (preds[0].name().unwrap(), preds[0].arity());
+
+        let (mut prev_preds, mut prev_queue) =
+            match self.term_dirs.get_old(name.clone(), arity).cloned() {
+                Some((preds, queue)) => (preds, queue),
+                None => (Predicate::new(), VecDeque::new()),
+            };
+
         let queue = self.rel_worker.parse_queue(&mut indices)?;
-        let result = (append_preds(preds), queue);
+
+        prev_preds.0.extend(preds.iter().cloned());
+        prev_queue.extend(queue.iter().cloned());
+
+        let result = (prev_preds, prev_queue);
+
+        self.term_dirs.add_new((name, arity), preds, queue);
 
         let in_situ_code_dir = &mut indices.term_stream.wam.indices.in_situ_code_dir;