refine EOF handling more (#1873)

src/parser/lexer.rs

@@ -24,9 +24,9 @@ macro_rules! consume_chars_with {
 }
 
 #[derive(Debug, Default)]
-pub struct LayoutInfo {
+struct LayoutInfo {
-    pub inserted: bool,
+    inserted: bool,
-    pub more: bool,
+    more: bool,
 }
 
 #[derive(Debug, PartialEq)]
@@ -909,7 +909,7 @@ impl<'a, R: CharRead> Lexer<'a, R> {
         }
     }
 
-    pub fn consume_layout(
+    fn consume_layout(
         &mut self,
         c: Option<char>,
         layout_info: &mut LayoutInfo,
@@ -938,20 +938,29 @@ impl<'a, R: CharRead> Lexer<'a, R> {
         Ok(())
     }
 
-    fn scan_for_layout(&mut self) -> Result<bool, ParserError> {
+    pub fn scan_for_layout(&mut self) -> Result<bool, ParserError> {
+        match self.lookahead_char() {
+            Err(e) => {
+                Err(e)
+            }
+            Ok(c) => {
                 let mut layout_info = LayoutInfo { inserted: false, more: true };
+                let mut cr = Some(c);
 
                 loop {
-            let cr = self.lookahead_char();
+                    self.consume_layout(cr, &mut layout_info)?;
-            self.consume_layout(cr.ok(), &mut layout_info)?;
 
                     if !layout_info.more {
                         break;
                     }
+
+                    cr = self.lookahead_char().ok();
                 }
 
                 Ok(layout_info.inserted)
             }
+        }
+    }
 
     pub fn next_token(&mut self) -> Result<Token, ParserError> {
         let layout_inserted = self.scan_for_layout()?;

src/parser/parser.rs

@@ -620,30 +620,6 @@ impl<'a, R: CharRead> Parser<'a, R> {
         false
     }
 
-    pub fn devour_whitespace(&mut self) -> Result<(), ParserError> {
-        match self.lexer.lookahead_char() {
-            Err(e) => { // if e.is_unexpected_eof() => {
-                return Err(e);
-            }
-            Ok(c) => {
-                let mut layout_info = LayoutInfo { inserted: false, more: true };
-                let mut cr = Some(c);
-
-                loop {
-                    self.lexer.consume_layout(cr, &mut layout_info)?;
-
-                    if !layout_info.more {
-                        break;
-                    }
-
-                    cr = self.lexer.lookahead_char().ok();
-                }
-            }
-        }
-
-        Ok(())
-    }
-
     pub fn reset(&mut self) {
         self.stack.clear()
     }

src/read.rs

@@ -25,12 +25,12 @@ use std::io::{Cursor, Error, ErrorKind, Read};
 type SubtermDeque = VecDeque<(usize, usize)>;
 
 pub(crate) fn devour_whitespace<'a, R: CharRead>(parser: &mut Parser<'a, R>) -> Result<bool, ParserError> {
-    match parser.devour_whitespace() {
+    match parser.lexer.scan_for_layout() {
         Err(e) if e.is_unexpected_eof() => {
             Ok(true)
         }
         Err(e) => Err(e),
-        Ok(()) => {
+        Ok(_) => {
             Ok(false)
         }
     }

src/toplevel.pl

@@ -152,6 +152,7 @@ expand_op_list([Op | OtherOps], Pred, Spec, [(:- op(Pred, Spec, Op)) | OtherResu
 
 
 read_and_match :-
+    '$debug_hook',
     '$read_query_term'(_, Term, _, _, VarList),
     instruction_match(Term, VarList).