reconcile '[]' and [] (#644), solve remaining conformity issues (#648), move (-->) from dcgs.pl to builtins.pl (#649)

Cargo.lock

@@ -502,6 +502,17 @@ version = "0.2.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005"
 
+[[package]]
+name = "libsodium-sys"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a685b64f837b339074115f2e7f7b431ac73681d08d75b389db7498b8892b8a58"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.3.4"
@@ -894,9 +905,9 @@ dependencies = [
 
 [[package]]
 name = "prolog_parser"
-version = "0.8.63"
+version = "0.8.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fa8dbe0881bcc9a247db279802701d87bbe9d4c6604bb0e5cad3dd3314f241d"
+checksum = "fdcfe8260335a150812e36b171ff985ce88f02de16c05f4a13fe4f10e8c247f8"
 dependencies = [
  "lexical",
  "num-rug-adapter",
@@ -1221,7 +1232,7 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "scryer-prolog"
-version = "0.8.126"
+version = "0.8.127"
 dependencies = [
  "base64 0.12.3",
  "blake2",
@@ -1250,6 +1261,7 @@ dependencies = [
  "rustyline",
  "select",
  "sha3",
+ "sodiumoxide",
  "unicode_reader",
 ]
 
@@ -1390,6 +1402,17 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4"
 
+[[package]]
+name = "sodiumoxide"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7038b67c941e23501573cb7242ffb08709abe9b11eb74bceff875bbda024a6a8"
+dependencies = [
+ "libc",
+ "libsodium-sys",
+ "serde",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "scryer-prolog"
-version = "0.8.126"
+version = "0.8.127"
 authors = ["Mark Thom <markjordanthom@gmail.com>"]
 edition = "2018"
 description = "A modern Prolog implementation written mostly in Rust."
@@ -32,7 +32,7 @@ libc = "0.2.62"
 nix = "0.15.0"
 num-rug-adapter = { optional = true, version = "0.1.3" }
 ordered-float = "0.5.0"
-prolog_parser = { version = "0.8.63", default-features = false }
+prolog_parser = { version = "0.8.65", default-features = false }
 ref_thread_local = "0.0.0"
 rug = { version = "1.4.0", optional = true }
 rustyline = "6.0.0"

src/heap_print.rs

@@ -1298,6 +1298,7 @@ impl<'a, Outputter: HCValueOutputter> HCPrinter<'a, Outputter> {
                     !iter.immediate_leaf_has_property(|addr, heap| {
                         match heap.index_addr(&addr).as_ref() {
                             &HeapCellValue::Integer(ref n) => &**n >= &0,
+                            &HeapCellValue::Addr(Addr::Fixnum(n)) => n >= 0,
                             &HeapCellValue::Addr(Addr::Float(f)) => f >= OrderedFloat(0f64),
                             &HeapCellValue::Rational(ref r) => &**r >= &0,
                             _ => false

src/lib/builtins.pl

@@ -24,6 +24,8 @@ user:term_expansion((:- op(Pred, Spec, [Op | OtherOps])), OpResults) :-
 :- op(400, yfx, [div, //, rdiv, <<, >>, mod, rem]).
 :- op(200, fy, [+, -, \]).
 
+:- op(1200, xfx, -->).
+
 % arithmetic comparison operators.
 :- op(700, xfx, [>, <, =\=, =:=, >=, =<]).
 

src/lib/dcgs.pl

@@ -1,7 +1,7 @@
-:- module(dcgs, [op(1200, xfx, -->),
+:- module(dcgs,
-		 op(1105, xfy, '|'),
+          [op(1105, xfy, '|'),
-		 phrase/2,
+		   phrase/2,
-		 phrase/3]).
+		   phrase/3]).
 
 :- use_module(library(error)).
 :- use_module(library(lists), [append/3]).

src/machine/machine_errors.rs

@@ -840,5 +840,4 @@ impl From<ParserError> for EvalSession {
     fn from(err: ParserError) -> Self {
         EvalSession::from(SessionError::ParserError(err))
     }
-
 }

src/machine/machine_state.rs

@@ -636,11 +636,10 @@ impl MachineState {
         }
 
         let mut orig_stream = stream.clone();
-        let mut stream = self.open_parsing_stream(stream, "read_term", 3)?;
 
         loop {
             match self.read(
-                &mut stream,
+                stream.clone(),
                 indices.atom_tbl.clone(),
                 &indices.op_dir,
             ) {
@@ -1337,14 +1336,8 @@ pub(crate) trait CallPolicy: Any + fmt::Debug {
                 return_from_clause!(machine_st.last_call, machine_st)
             }
             &BuiltInClauseType::Read => {
-                let mut stream = machine_st.open_parsing_stream(
-                    current_input_stream.clone(),
-                    "read",
-                    1,
-                )?;
-
                 match machine_st.read(
-                    &mut stream,
+                    current_input_stream.clone(),
                     indices.atom_tbl.clone(),
                     &indices.op_dir,
                 ) {

src/machine/streams.rs

@@ -6,13 +6,15 @@ use crate::machine::machine_errors::*;
 use crate::machine::machine_indices::*;
 use crate::machine::machine_state::*;
 
-use std::cmp::Ordering;
+use std::cmp::{min, Ordering};
 use std::cell::RefCell;
 use std::error::Error;
 use std::fmt;
 use std::fs::File;
+use std::io;
 use std::io::{stdout, Cursor, ErrorKind, Read, Seek, SeekFrom, Write};
 use std::hash::{Hash, Hasher};
+use std::mem;
 use std::net::{Shutdown, TcpStream};
 use std::ops::DerefMut;
 use std::rc::Rc;
@@ -92,20 +94,83 @@ impl EOFAction {
     }
 }
 
+fn parser_top_to_bytes(mut buf: Vec<io::Result<char>>) -> io::Result<Vec<u8>> {
+    let mut str_buf = String::new();
+
+    while let Some(c) = buf.pop() {
+        str_buf.push(c?);
+    }
+
+    unsafe {
+        let array = str_buf.as_bytes_mut();
+        array.reverse();
+        Ok(Vec::from(array))
+    }
+}
+
 /* all these streams are closed automatically when the instance is
  * dropped. */
 pub enum StreamInstance {
     Bytes(Cursor<Vec<u8>>),
-    DynReadSource(Box<dyn Read>),
     InputFile(ClauseName, File),
     OutputFile(ClauseName, File, bool), // File, append.
     Null,
+    PausedPrologStream(Vec<u8>, Box<StreamInstance>),
     ReadlineStream(ReadlineStream),
+    StaticStr(Cursor<&'static str>),
     Stdout,
     TcpStream(ClauseName, TcpStream),
     TlsStream(ClauseName, TlsStream<TcpStream>)
 }
 
+impl StreamInstance {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match self {
+            StreamInstance::PausedPrologStream(ref mut put_back, ref mut stream) => {
+                let mut index = 0;
+
+                while index < min(buf.len(), put_back.len()) {
+                    let b = put_back.pop().unwrap();
+                    buf[index] = b;
+                    index += 1;
+                }
+
+                if index == buf.len() {
+                    Ok(buf.len())
+                } else {
+                    stream.read(&mut buf[index ..])
+                }
+            }
+            StreamInstance::InputFile(_, ref mut file) => {
+                file.read(buf)
+            }
+            StreamInstance::TcpStream(_, ref mut tcp_stream) => {
+                tcp_stream.read(buf)
+            }
+            StreamInstance::TlsStream(_, ref mut tls_stream) => {
+                tls_stream.read(buf)
+            }
+            StreamInstance::ReadlineStream(ref mut rl_stream) => {
+                rl_stream.read(buf)
+            }
+            StreamInstance::StaticStr(ref mut src) => {
+                src.read(buf)
+            }
+            StreamInstance::Bytes(ref mut cursor) => {
+                cursor.read(buf)
+            }
+            StreamInstance::OutputFile(..) |
+            StreamInstance::Stdout |
+            StreamInstance::Null => {
+                Err(std::io::Error::new(
+                    ErrorKind::PermissionDenied,
+                    StreamError::ReadFromOutputStream,
+                ))
+            }
+        }
+    }
+}
+
 impl Drop for StreamInstance {
     fn drop(&mut self) {
         match self {
@@ -126,15 +191,20 @@ impl fmt::Debug for StreamInstance {
         match self {
             &StreamInstance::Bytes(ref bytes) =>
                 write!(fmt, "Bytes({:?})", bytes),
-            &StreamInstance::DynReadSource(_) =>
+            &StreamInstance::StaticStr(_) =>
-                write!(fmt, "DynReadSource(_)"),  // Hacky solution.
+                write!(fmt, "StaticStr(_)"),  // Hacky solution.
-            &StreamInstance::InputFile(_, ref file) => write!(fmt, "InputFile({:?})", file),
+            &StreamInstance::InputFile(_, ref file) =>
-            &StreamInstance::OutputFile(_, ref file, _) => write!(fmt, "OutputFile({:?})", file),
+                write!(fmt, "InputFile({:?})", file),
-            &StreamInstance::Null => write!(fmt, "Null"),
+            &StreamInstance::OutputFile(_, ref file, _) =>
+                write!(fmt, "OutputFile({:?})", file),
+            &StreamInstance::Null =>
+                write!(fmt, "Null"),
+            &StreamInstance::PausedPrologStream(ref put_back, ref stream) =>
+                write!(fmt, "PausedPrologStream({:?}, {:?})", put_back, stream),
             &StreamInstance::ReadlineStream(ref readline_stream) =>
                 write!(fmt, "ReadlineStream({:?})", readline_stream),
-            // &StreamInstance::Stdin => write!(fmt, "Stdin"),
+            &StreamInstance::Stdout =>
-            &StreamInstance::Stdout => write!(fmt, "Stdout"),
+                write!(fmt, "Stdout"),
             &StreamInstance::TcpStream(_, ref tcp_stream) =>
                 write!(fmt, "TcpStream({:?})", tcp_stream),
             &StreamInstance::TlsStream(_, ref tls_stream) =>
@@ -282,7 +352,7 @@ impl From<ReadlineStream> for Stream {
 
 impl From<&'static str> for Stream {
     fn from(src: &'static str) -> Stream {
-        Stream::from_inst(StreamInstance::DynReadSource(Box::new(src.as_bytes())))
+        Stream::from_inst(StreamInstance::StaticStr(Cursor::new(src)))
     }
 }
 
@@ -413,8 +483,9 @@ impl Stream {
     fn mode(&self) -> &'static str {
         match self.stream_inst.0.borrow().1 {
             StreamInstance::Bytes(_) |
+            StreamInstance::PausedPrologStream(..) |
             StreamInstance::ReadlineStream(_) |
-            StreamInstance::DynReadSource(_) |
+            StreamInstance::StaticStr(_) |
             StreamInstance::InputFile(..) => {
                 "read"
             }
@@ -493,7 +564,6 @@ impl Stream {
     pub(crate)
     fn is_stdin(&self) -> bool {
         match self.stream_inst.0.borrow().1 {
-            //StreamInstance::Stdin |
             StreamInstance::ReadlineStream(_) => {
                 true
             }
@@ -523,12 +593,12 @@ impl Stream {
     pub(crate)
     fn is_input_stream(&self) -> bool {
         match self.stream_inst.0.borrow().1 {
-            // StreamInstance::Stdin |
             StreamInstance::TcpStream(..) |
             StreamInstance::TlsStream(..) |
             StreamInstance::Bytes(_) |
+            StreamInstance::PausedPrologStream(..) |
             StreamInstance::ReadlineStream(_) |
-            StreamInstance::DynReadSource(_) |
+            StreamInstance::StaticStr(_) |
             StreamInstance::InputFile(..) => {
                 true
             }
@@ -555,27 +625,49 @@ impl Stream {
         }
     }
 
+    fn unpause_stream(&mut self) {
+        let stream_inst =
+            match self.stream_inst.0.borrow_mut().1 {
+                StreamInstance::PausedPrologStream(ref put_back, ref mut stream_inst)
+                    if put_back.is_empty() => {
+                        mem::replace(&mut **stream_inst, StreamInstance::Null)
+                    }
+                _ => {
+                    return;
+                }
+            };
+
+        self.stream_inst.0.borrow_mut().1 = stream_inst;
+    }
+
     // returns true on success.
     #[inline]
     pub(super)
     fn reset(&mut self) -> bool {
         self.stream_inst.0.borrow_mut().0 = false;
 
-        match self.stream_inst.0.borrow_mut().1 {
+        loop {
-            StreamInstance::Bytes(ref mut cursor) => {
+            match self.stream_inst.0.borrow_mut().1 {
-                cursor.set_position(0);
+                StreamInstance::Bytes(ref mut cursor) => {
-                true
+                    cursor.set_position(0);
-            }
+                    return true;
-            StreamInstance::InputFile(_, ref mut file) => {
+                }
-                file.seek(SeekFrom::Start(0)).unwrap();
+                StreamInstance::InputFile(_, ref mut file) => {
-                true
+                    file.seek(SeekFrom::Start(0)).unwrap();
-            }
+                    return true;
-            StreamInstance::ReadlineStream(_) => {
+                }
-                true
+                StreamInstance::PausedPrologStream(ref mut put_back, _) => {
-            }
+                    put_back.clear();
-            _ => {
+                }
-                false
+                StreamInstance::ReadlineStream(_) => {
+                    return true;
+                }
+                _ => {
+                    return false;
+                }
             }
+
+            self.unpause_stream();
         }
     }
 
@@ -687,6 +779,34 @@ impl Stream {
             }
         }
     }
+
+    #[inline]
+    pub(crate)
+    fn pause_stream(&mut self, buf: Vec<io::Result<char>>) -> io::Result<()> {
+        match self.stream_inst.0.borrow_mut().1 {
+            StreamInstance::PausedPrologStream(ref mut inner_buf, _) => {
+                inner_buf.extend(parser_top_to_bytes(buf)?.into_iter());
+                return Ok(());
+            }
+            _ => {
+            }
+        }
+
+        if !buf.is_empty() {
+            let stream_inst = mem::replace(
+                &mut self.stream_inst.0.borrow_mut().1,
+                StreamInstance::Null,
+            );
+
+            self.stream_inst.0.borrow_mut().1 =
+                StreamInstance::PausedPrologStream(
+                    parser_top_to_bytes(buf)?,
+                    Box::new(stream_inst),
+                );
+        }
+
+        Ok(())
+    }
 }
 
 impl MachineState {
@@ -883,7 +1003,7 @@ impl MachineState {
         stub_name: &'static str,
         stub_arity: usize,
     ) -> Result<PrologStream, MachineStub> {
-        match parsing_stream(stream.clone()) {
+        match parsing_stream(stream) {
             Ok(parsing_stream) => {
                 Ok(parsing_stream)
             }
@@ -1045,38 +1165,11 @@ impl MachineState {
 }
 
 impl Read for Stream {
+    #[inline]
     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        match self.stream_inst.0.borrow_mut().1 {
+        let bytes_read = self.stream_inst.0.borrow_mut().1.read(buf)?;
-            StreamInstance::InputFile(_, ref mut file) => {
+        self.unpause_stream();
-                file.read(buf)
+        Ok(bytes_read)
-            }
-            StreamInstance::TcpStream(_, ref mut tcp_stream) => {
-                tcp_stream.read(buf)
-            }
-            StreamInstance::TlsStream(_, ref mut tls_stream) => {
-                tls_stream.read(buf)
-            }
-            StreamInstance::ReadlineStream(ref mut rl_stream) => {
-                rl_stream.read(buf)
-            }
-            StreamInstance::DynReadSource(ref mut src) => {
-                src.read(buf)
-            }
-            StreamInstance::Bytes(ref mut cursor) => {
-                cursor.read(buf)
-            }
-/*
-            StreamInstance::Stdin => {
-                stdin().read(buf)
-            }
-*/
-            StreamInstance::OutputFile(..) | StreamInstance::Stdout | StreamInstance::Null => {
-                Err(std::io::Error::new(
-                    ErrorKind::PermissionDenied,
-                    StreamError::ReadFromOutputStream,
-                ))
-            }
-        }
     }
 }
 
@@ -1098,8 +1191,11 @@ impl Write for Stream {
             StreamInstance::Stdout => {
                 stdout().write(buf)
             }
-            StreamInstance::DynReadSource(_) | StreamInstance::ReadlineStream(_) |
+            StreamInstance::PausedPrologStream(..) |
-            StreamInstance::InputFile(..) | StreamInstance::Null => {
+            StreamInstance::StaticStr(_) |
+            StreamInstance::ReadlineStream(_) |
+            StreamInstance::InputFile(..) |
+            StreamInstance::Null => {
                 Err(std::io::Error::new(
                     ErrorKind::PermissionDenied,
                     StreamError::WriteToInputStream,
@@ -1125,8 +1221,11 @@ impl Write for Stream {
             StreamInstance::Stdout => {
                 stdout().flush()
             }
-            StreamInstance::DynReadSource(_) | StreamInstance::ReadlineStream(_) |
+            StreamInstance::PausedPrologStream(..) |
-            StreamInstance::InputFile(..) | StreamInstance::Null => {
+            StreamInstance::StaticStr(_) |
+            StreamInstance::ReadlineStream(_) |
+            StreamInstance::InputFile(..) |
+            StreamInstance::Null => {
                 Err(std::io::Error::new(
                     ErrorKind::PermissionDenied,
                     StreamError::FlushToInputStream,

src/machine/system_calls.rs

@@ -2470,10 +2470,11 @@ impl MachineState {
                         string.push(c as char);
                     }
                 } else {
-                    let mut iter = self.open_parsing_stream(stream.clone(),
+                    let mut iter = self.open_parsing_stream(
-                                                            "get_n_chars",
+                        stream.clone(),
-                                                            2,
+                        "get_n_chars",
-                                                            )?;
+                        2,
+                    )?;
 
                     for _ in 0..num {
                          let result = iter.next();
@@ -4475,16 +4476,10 @@ impl MachineState {
                 let mut heap_pstr_iter = self.heap_pstr_iter(self[temp_v!(1)]);
                 let chars = heap_pstr_iter.to_string();
 
-                let mut stream = self.open_parsing_stream(
-                    Stream::from(chars),
-                    "read_term_from_chars",
-                    2,
-                )?;
-
                 if let Addr::EmptyList = heap_pstr_iter.focus() {
                     let term_write_result =
                         match self.read(
-                            &mut stream,
+                            Stream::from(chars),
                             indices.atom_tbl.clone(),
                             &indices.op_dir,
                         ) {

src/machine/term_expansion.rs

@@ -165,7 +165,7 @@ impl<'a> TermStream<'a> {
 
     #[inline]
     pub fn eof(&mut self) -> Result<bool, ParserError> {
-	self.parser.devour_whitespace()?; // eliminate dangling comments before checking for EOF.
+	    self.parser.devour_whitespace()?; // eliminate dangling comments before checking for EOF.
         Ok(self.stack.is_empty() && self.parser.eof()?)
     }
 

src/read.rs

@@ -67,10 +67,10 @@ pub mod readline {
                         }
                     }
 
-                    if self.pending_input.get_ref().chars().last() != Some('\n')
+                    if self.pending_input.get_ref().chars().last() != Some('\n') {
-                    {
                         *self.pending_input.get_mut() += "\n";
                     }
+
                     self.pending_input.read(buf)
                 }
                 Err(ReadlineError::Eof) => {
@@ -160,12 +160,23 @@ pub mod readline {
 impl MachineState {
     pub fn read(
         &mut self,
-        inner: &mut PrologStream,
+        mut inner: Stream,
         atom_tbl: TabledData<Atom>,
         op_dir: &OpDir,
     ) -> Result<TermWriteResult, ParserError> {
-        let mut parser = Parser::new(inner, atom_tbl, self.flags);
+        let mut stream = parsing_stream(inner.clone())?;
-        let term = parser.read_term(composite_op!(op_dir))?;
+
+        let term = {
+            let mut parser = Parser::new(&mut stream, atom_tbl, self.flags);
+            parser.read_term(composite_op!(op_dir))?
+        };
+
+        // 'pausing' the stream saves the pending top buffer
+        // created by the parsing stream, which was created in this
+        // scope and is about to be destroyed in it.
+
+        let buf = stream.take_buf();
+        inner.pause_stream(buf)?;
 
         Ok(write_term_to_heap(&term, self))
     }

src/toplevel.pl

@@ -108,28 +108,28 @@ read_and_match :-
     '$read_query_term'(_, Term, _, _, VarList),
     instruction_match(Term, VarList).
 
+
 % make compile_batch, a system routine, callable.
 compile_batch :- '$compile_batch'.
 
 instruction_match(Term, VarList) :-
     (  var(Term) ->
        throw(error(instantiation_error, repl/0))
-    ;
+    ;  Term = [Item] ->
-    Term = [Item] -> !,
+       !,
-                     (  atom(Item) ->
+       (  atom(Item) ->
-	                    (  Item == user ->
+	      (  Item == user ->
-	                       catch(compile_batch, E, print_exception_with_check(E))
+	         catch(compile_batch, E, print_exception_with_check(E))
-	                    ;  consult(Item)
+	      ;  consult(Item)
-	                    )
+	      )
-                     ;
+       ;
-	                 catch(throw(error(type_error(atom, Item), repl/0)),
+	   catch(throw(error(type_error(atom, Item), repl/0)),
-		                   E,
+		     E,
-		                   print_exception_with_check(E))
+		     print_exception_with_check(E))
-                     )
+       )
-    ;
+    ;  Term = end_of_file ->
-    Term = end_of_file -> halt
+       halt
-    ;
+    ;  submit_query_and_print_results(Term, VarList)
-    submit_query_and_print_results(Term, VarList)
     ).
 
 :- use_module(library(iso_ext)).
@@ -159,10 +159,10 @@ needs_bracketing(Value, Op) :-
 	      false),
     (  EqPrec < FPrec ->
        true
-    ;  '$quoted_token'(F) ->
-       true
     ;  FPrec > 0, F == Value, graphic_token_char(F) ->
        true
+    ;  F \== '.', '$quoted_token'(F) ->
+       true
     ;  EqPrec == FPrec,
        memberchk(EqSpec, [fx,xfx,yfx])
     ).