From b767e4de25a5f62591140a0c25153dc4959805d9 Mon Sep 17 00:00:00 2001 From: Dan Rose Date: Tue, 13 May 2025 15:00:00 -0500 Subject: [PATCH 01/10] xor is not a standard operator --- src/lib/ops_and_meta_predicates.pl | 1 - src/tests/arithmetic.pl | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/ops_and_meta_predicates.pl b/src/lib/ops_and_meta_predicates.pl index 8f904d63..8ffdfb03 100644 --- a/src/lib/ops_and_meta_predicates.pl +++ b/src/lib/ops_and_meta_predicates.pl @@ -26,7 +26,6 @@ :- op(200, xfy, ^). :- op(500, yfx, /\). :- op(500, yfx, \/). -:- op(500, yfx, xor). :- op(400, yfx, div). :- op(400, yfx, //). :- op(400, yfx, rdiv). diff --git a/src/tests/arithmetic.pl b/src/tests/arithmetic.pl index 2e4872d4..eea4b3ac 100644 --- a/src/tests/arithmetic.pl +++ b/src/tests/arithmetic.pl @@ -258,14 +258,14 @@ test_and_or_xor(X, Y, AndExpected, OrExpected, XorExpected) :- Or2 is Y \/ X, Or == OrExpected, Or == Or2, - Xor is X xor Y, - Xor2 is Y xor X, + Xor is xor(X, Y), + Xor2 is xor(Y, X), Xor == XorExpected, Xor2 == Xor, call(is, And3, X /\ Y), call(is, Or3, X \/ Y), - call(is, Xor3, X xor Y), + call(is, Xor3, xor(X, Y)), And3 == And, Or3 == Or, Xor3 == Xor. @@ -613,7 +613,7 @@ test("and_or_xor", ( ]), arithmetic_tests:test_and_or_xor(X, Y, AndExpected, OrExpected, XorExpected)), \+ catch(_ is 1 /\ 2.0, _, false), \+ catch(_ is 1 \/ 2.0, _, false), - \+ catch(_ is 1 xor 2.0, _, false) + \+ catch(_ is xor(1, 2.0), _, false) )). test("mod_rem", ( From 9456517bed1a9648d835b9c6954dc793e32dbebe Mon Sep 17 00:00:00 2001 From: Dan Rose Date: Thu, 22 May 2025 13:57:55 -0500 Subject: [PATCH 02/10] Remove obsolete compatibility for Rust<1.80 --- Cargo.toml | 1 - src/machine/mod.rs | 26 -------------------------- 2 files changed, 27 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ddcf79fd..e7b7f930 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,6 @@ hostname = ["dep:hostname"] tls = ["dep:native-tls"] http = ["dep:warp", "dep:reqwest"] crypto-full = [] -"rust-version-1.80" = [] [build-dependencies] indexmap = "2.3.0" diff --git a/src/machine/mod.rs b/src/machine/mod.rs index da2ab6df..d180f02d 100644 --- a/src/machine/mod.rs +++ b/src/machine/mod.rs @@ -118,32 +118,6 @@ fn current_dir() -> PathBuf { } } -#[cfg(not(feature = "rust-version-1.80"))] -mod libraries { - use indexmap::IndexMap; - use std::sync::OnceLock; - - fn libraries() -> &'static IndexMap<&'static str, &'static str> { - static LIBRARIES: OnceLock> = OnceLock::new(); - LIBRARIES.get_or_init(|| { - let mut m = IndexMap::new(); - - include!(concat!(env!("OUT_DIR"), "/libraries.rs")); - - m - }) - } - - pub(crate) fn contains(name: &str) -> bool { - libraries().contains_key(name) - } - - pub(crate) fn get(name: &str) -> Option<&'static str> { - libraries().get(name).copied() - } -} - -#[cfg(feature = "rust-version-1.80")] mod libraries { use indexmap::IndexMap; use std::sync::LazyLock; From 37ac5bb3c1bcb4e7d973c9074cebf3536c2bb6de Mon Sep 17 00:00:00 2001 From: notoria Date: Sun, 25 May 2025 13:37:28 +0200 Subject: [PATCH 03/10] Strengthened `(mod)/2` in CLP(Z) ?- Z = 8, X = 9, #Z #= #X mod #Y. Z = 8, X = 9, clpz:(Y in 9..sup), clpz:(9 mod#Y#=8). % unexpected Expected: false --- src/lib/clpz.pl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lib/clpz.pl b/src/lib/clpz.pl index b14546bc..9daad75c 100644 --- a/src/lib/clpz.pl +++ b/src/lib/clpz.pl @@ -5088,11 +5088,21 @@ run_propagator(pmod(X,Y,Z), MState) --> ; nonvar(Z), nonvar(X) -> ( Z > 0 -> ( X < 0 -> true - ; X >= Z + ; X >= Z, + % due to X = Z+Y*_ and Y > Z + ( X-Z > 0 -> + X-Z > Z + ; true + ) ) ; Z < 0 -> ( X > 0 -> true - ; X =< Z + ; X =< Z, + % due to X = Z+Y*_ and Y < Z + ( X-Z < 0 -> + X-Z < Z + ; true + ) ) ; Z =:= 0 % Multiple solutions so do nothing special. ), From 4a6f86591cc8c5108910e4117b47395b2a817799 Mon Sep 17 00:00:00 2001 From: notoria Date: Sun, 25 May 2025 13:38:56 +0200 Subject: [PATCH 04/10] Strengthened `(mod)/2` in CLP(Z) ?- 0 #= 2 mod #Y. clpz:(Y in inf.. -1\/1..sup), clpz:(2 mod#Y#=0). % unexpected Expected: clpz:(Y in-2.. -1\/1..2), clpz:(2 mod Y#=0) --- src/lib/clpz.pl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/lib/clpz.pl b/src/lib/clpz.pl index 9daad75c..0f383094 100644 --- a/src/lib/clpz.pl +++ b/src/lib/clpz.pl @@ -5122,7 +5122,16 @@ run_propagator(pmod(X,Y,Z), MState) --> domain_remove_greater_than(YD, YMax, YD1) }, fd_put(Y, YD1, YPs) % queue_goal(Y #< Z) - ; true + ; Z =:= 0, + ( X =:= 0 -> + kill(MState) % trivial + ; % only 4 solutions {-abs(X),-1,1,abs(X)} + { YL is -abs(X), YU is abs(X), + fd_get(Y, YD0, YPs), + domain_remove_smaller_than(YD0, YL, YD1), + domain_remove_greater_than(YD1, YU, YD) }, + fd_put(Y, YD, YPs) + ) ) ) ; run_propagator(pmodz(X,Y,Z), MState), From 4b480a7eab68b4e1dc8d07c36ae7b1645ee073bb Mon Sep 17 00:00:00 2001 From: notoria Date: Sun, 25 May 2025 13:39:31 +0200 Subject: [PATCH 05/10] Strengthened `(mod)/2` in CLP(Z) ?- 8 #= -1 mod #Y. clpz:(Y in 9..sup), clpz:(-1 mod#Y#=8). % unexpected Expected: Y = 9 ?- 8 #= -12 mod #Y. clpz:(Y in 9..sup), clpz:(-12 mod#Y#=8). % unexpected Expected `Y` with finite domain --- src/lib/clpz.pl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/lib/clpz.pl b/src/lib/clpz.pl index 0f383094..89019b4f 100644 --- a/src/lib/clpz.pl +++ b/src/lib/clpz.pl @@ -5110,7 +5110,23 @@ run_propagator(pmod(X,Y,Z), MState) --> YU < X, X =< 0 } -> kill(MState), Z =:= X ; { fd_get(Y, _, n(YL), _, _), YL > X, X >= 0 } -> kill(MState), Z =:= X - ; ( Z > 0 -> + ; ( Z > 0, X < 0 -> + { fd_get(Y, YD, YPs), + YMin is Z+1, + YMax is Z-X, + domain_remove_smaller_than(YD, YMin, YD1), + domain_remove_greater_than(YD1, YMax, YD2) }, + fd_put(Y, YD2, YPs) + % queue_goal((Y #> Z, Y #=< Z-X)) + ; Z < 0, X > 0 -> + { fd_get(Y, YD, YPs), + YMax is Z-1, + YMin is Z-X, + domain_remove_greater_than(YD, YMax, YD1), + domain_remove_smaller_than(YD1, YMin, YD2) }, + fd_put(Y, YD2, YPs) + % queue_goal((Y #< Z, Y #>= Z-X)) + ; Z > 0 -> { fd_get(Y, YD, YPs), YMin is Z + 1, domain_remove_smaller_than(YD, YMin, YD1) }, From 17377064cf3379255afbdf0b355108adbb45cc5d Mon Sep 17 00:00:00 2001 From: Markus Triska Date: Sat, 14 Jun 2025 22:43:35 +0200 Subject: [PATCH 06/10] use nonterminal indicators, correct meta_predicate/1 for higher-order phrase//N Noted by @UWN in https://github.com/mthom/scryer-prolog/discussions/2872. Many thanks! --- src/lib/dcgs.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/dcgs.pl b/src/lib/dcgs.pl index 2dc858a4..835e80a5 100644 --- a/src/lib/dcgs.pl +++ b/src/lib/dcgs.pl @@ -12,8 +12,8 @@ to learn more about them. [op(1105, xfy, '|'), phrase/2, phrase/3, - phrase/4, - phrase/5, + phrase//2, + phrase//3, seq//1, seqq//1, ... //0, @@ -29,9 +29,9 @@ to learn more about them. :- meta_predicate phrase(2, ?, ?). -:- meta_predicate phrase(2, ?, ?, ?). +:- meta_predicate(phrase(3, ?, ?, ?)). -:- meta_predicate phrase(2, ?, ?, ?, ?). +:- meta_predicate(phrase(4, ?, ?, ?, ?)). :- meta_predicate(','(2, 2, ?, ?)). From 8e01eea440d72142ec2a03325a6ff371b479a440 Mon Sep 17 00:00:00 2001 From: Mark Thom Date: Tue, 1 Jul 2025 18:40:49 -0700 Subject: [PATCH 07/10] be more careful about concatenating '.' in parse_number_from_string (#2986) --- src/machine/system_calls.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/machine/system_calls.rs b/src/machine/system_calls.rs index dc5da32a..9d056ff9 100644 --- a/src/machine/system_calls.rs +++ b/src/machine/system_calls.rs @@ -911,13 +911,13 @@ impl MachineState { use crate::parser::lexer::*; let nx = self.store(self.deref(self.registers[2])); - let add_dot = !string.ends_with('.'); + let add_dot = !string.ends_with('.') || string.ends_with("'."); let cursor = std::io::Cursor::new(string); let iter = std::io::Read::chain(cursor, { let mut dot_buf: [u8; '.'.len_utf8()] = [0u8]; - if add_dot { + if add_dot && !string.ends_with('\'') { '.'.encode_utf8(&mut dot_buf); } From ce56b5ed6b28745b202ef939dd227aa9491b2721 Mon Sep 17 00:00:00 2001 From: Mark Thom Date: Sat, 5 Jul 2025 00:58:28 -0700 Subject: [PATCH 08/10] modify number_token to produce partial lexing results (#2986) --- src/machine/system_calls.rs | 17 +- src/parser/lexer.rs | 337 +++++++++++++++--------------------- 2 files changed, 144 insertions(+), 210 deletions(-) diff --git a/src/machine/system_calls.rs b/src/machine/system_calls.rs index 9d056ff9..f14645e7 100644 --- a/src/machine/system_calls.rs +++ b/src/machine/system_calls.rs @@ -911,27 +911,16 @@ impl MachineState { use crate::parser::lexer::*; let nx = self.store(self.deref(self.registers[2])); - let add_dot = !string.ends_with('.') || string.ends_with("'."); - let cursor = std::io::Cursor::new(string); - - let iter = std::io::Read::chain(cursor, { - let mut dot_buf: [u8; '.'.len_utf8()] = [0u8]; - - if add_dot && !string.ends_with('\'') { - '.'.encode_utf8(&mut dot_buf); - } - - std::io::Cursor::new(dot_buf) - }); + let iter = std::io::Cursor::new(string); let mut lexer = Lexer::new(CharReader::new(iter), self); let mut tokens = vec![]; - match lexer.next_token() { + match lexer.next_number_token() { Ok(token @ Token::Literal(Literal::Atom(atom!("-")) | Literal::Char('-'))) => { tokens.push(token); - if let Ok(token) = lexer.next_token() { + if let Ok(token) = lexer.next_number_token() { tokens.push(token); } } diff --git a/src/parser/lexer.rs b/src/parser/lexer.rs index 06f8a326..75700f38 100644 --- a/src/parser/lexer.rs +++ b/src/parser/lexer.rs @@ -1,3 +1,5 @@ +use crate::arena::F64Ptr; +use crate::arena::TypedArenaPtr; use crate::atom_table::*; pub use crate::machine::machine_state::*; use crate::parser::ast::*; @@ -49,6 +51,55 @@ impl Token { } } +#[derive(Debug)] +enum Number { + BigInt(TypedArenaPtr), + Fixnum(Fixnum), + Float(F64Ptr), +} + +impl Number { + #[inline] + fn to_literal(self) -> Literal { + match self { + Number::BigInt(ibig) => Literal::Integer(ibig), + Number::Fixnum(fixnum) => Literal::Fixnum(fixnum), + Number::Float(f) => Literal::Float(f.as_offset()), + } + } +} + +#[derive(Debug)] +enum NumberToken { + Number(Number), + Partial(String), +} + +impl NumberToken { + #[inline] + fn to_token(self) -> Option { + match self { + NumberToken::Number(number) => Some(Token::Literal(number.to_literal())), + NumberToken::Partial(_) => None, + } + } +} + +macro_rules! try_nt { + ($token:expr, $e: expr) => {{ + match $e { + Ok(c) => c, + Err(e) => { + return if e.is_unexpected_eof() { + Ok(NumberToken::Partial($token)) + } else { + Err(e) + } + } + } + }}; +} + pub struct Lexer<'a, R> { pub(crate) reader: R, pub(crate) machine_st: &'a mut MachineState, @@ -443,7 +494,7 @@ impl<'a, R: CharRead> Lexer<'a, R> { } } - fn hexadecimal_constant(&mut self, start: char) -> Result { + fn hexadecimal_constant(&mut self, start: char) -> Result { self.skip_char(start); let mut c = self.lookahead_char()?; @@ -454,31 +505,21 @@ impl<'a, R: CharRead> Lexer<'a, R> { if hexadecimal_digit_char!(c) { self.skip_char(c); token.push(c); - c = self.lookahead_char()?; + c = try_nt!(token, self.lookahead_char()); } else { break; } } - i64::from_str_radix(&token, 16) - .map(|n| Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena))) - .or_else(|_| { - Integer::from_str_radix(&token, 16) - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) - }) + self.parse_integer_by_radix(&token, 16) + .map(NumberToken::Number) } else { self.return_char(start); Err(ParserError::ParseBigInt(self.line_num, self.col_num)) } } - fn octal_constant(&mut self, start: char) -> Result { + fn octal_constant(&mut self, start: char) -> Result { self.skip_char(start); let mut c = self.lookahead_char()?; @@ -489,31 +530,21 @@ impl<'a, R: CharRead> Lexer<'a, R> { if octal_digit_char!(c) { self.skip_char(c); token.push(c); - c = self.lookahead_char()?; + c = try_nt!(token, self.lookahead_char()); } else { break; } } - i64::from_str_radix(&token, 8) - .map(|n| Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena))) - .or_else(|_| { - Integer::from_str_radix(&token, 8) - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) - }) + self.parse_integer_by_radix(&token, 8) + .map(NumberToken::Number) } else { self.return_char(start); Err(ParserError::ParseBigInt(self.line_num, self.col_num)) } } - fn binary_constant(&mut self, start: char) -> Result { + fn binary_constant(&mut self, start: char) -> Result { self.skip_char(start); let mut c = self.lookahead_char()?; @@ -524,24 +555,14 @@ impl<'a, R: CharRead> Lexer<'a, R> { if binary_digit_char!(c) { self.skip_char(c); token.push(c); - c = self.lookahead_char()?; + c = try_nt!(token, self.lookahead_char()); } else { break; } } - i64::from_str_radix(&token, 2) - .map(|n| Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena))) - .or_else(|_| { - Integer::from_str_radix(&token, 2) - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) - }) + self.parse_integer_by_radix(&token, 2) + .map(NumberToken::Number) } else { self.return_char(start); Err(ParserError::ParseBigInt(self.line_num, self.col_num)) @@ -634,15 +655,10 @@ impl<'a, R: CharRead> Lexer<'a, R> { } } - fn vacate_with_float(&mut self, mut token: String) -> Result { + fn vacate_with_float(&mut self, mut token: String) -> Result { self.return_char(token.pop().unwrap()); - let n = parse_float_lossy(&token)?; - - Ok(Token::Literal(Literal::from(float_alloc!( - n, - self.machine_st.arena - )))) + Ok(Number::Float(float_alloc!(n, self.machine_st.arena))) } fn skip_underscore_in_number(&mut self) -> Result { @@ -663,17 +679,43 @@ impl<'a, R: CharRead> Lexer<'a, R> { } } - pub fn number_token(&mut self, leading_c: char) -> Result { + fn parse_integer_by_radix( + &mut self, + token: &String, + radix: u32, + ) -> Result { + i64::from_str_radix(&token, radix) + .map(|n| { + Fixnum::build_with_checked(n) + .map(Number::Fixnum) + .unwrap_or_else(|_| { + Number::BigInt(arena_alloc!(Integer::from(n), &mut self.machine_st.arena)) + }) + }) + .or_else(|_| { + token + .parse::() + .map(|n| Number::BigInt(arena_alloc!(n, &mut self.machine_st.arena))) + .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) + }) + } + + #[inline] + fn parse_integer(&mut self, token: &String) -> Result { + self.parse_integer_by_radix(token, 10) + } + + fn number_token(&mut self, leading_c: char) -> Result { let mut token = String::with_capacity(16); self.skip_char(leading_c); token.push(leading_c); - let mut c = self.skip_underscore_in_number()?; + let mut c = try_nt!(token, self.skip_underscore_in_number()); while decimal_digit_char!(c) { token.push(c); self.skip_char(c); - c = self.skip_underscore_in_number()?; + c = try_nt!(token, self.skip_underscore_in_number()); } if decimal_point_char!(c) { @@ -681,31 +723,17 @@ impl<'a, R: CharRead> Lexer<'a, R> { if self.reader.peek_char().is_none() { self.return_char('.'); - - token - .parse::() - .map(|n| Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena))) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) - }) + self.parse_integer(&token).map(NumberToken::Number) } else if decimal_digit_char!(self.lookahead_char()?) { token.push('.'); token.push(self.read_char()?); - let mut c = self.lookahead_char()?; + let mut c = try_nt!(token, self.lookahead_char()); while decimal_digit_char!(c) { token.push(c); self.skip_char(c); - c = self.lookahead_char()?; + c = try_nt!(token, self.lookahead_char()); } if exponent_char!(c) { @@ -713,12 +741,12 @@ impl<'a, R: CharRead> Lexer<'a, R> { token.push(c); let c = match self.lookahead_char() { - Err(_) => return self.vacate_with_float(token), + Err(_) => return self.vacate_with_float(token).map(NumberToken::Number), Ok(c) => c, }; if !sign_char!(c) && !decimal_digit_char!(c) { - return self.vacate_with_float(token); + return self.vacate_with_float(token).map(NumberToken::Number); } if sign_char!(c) { @@ -728,14 +756,14 @@ impl<'a, R: CharRead> Lexer<'a, R> { let c = match self.lookahead_char() { Err(_) => { self.return_char(token.pop().unwrap()); - return self.vacate_with_float(token); + return self.vacate_with_float(token).map(NumberToken::Number); } Ok(c) => c, }; if !decimal_digit_char!(c) { self.return_char(token.pop().unwrap()); - return self.vacate_with_float(token); + return self.vacate_with_float(token).map(NumberToken::Number); } } @@ -746,7 +774,7 @@ impl<'a, R: CharRead> Lexer<'a, R> { token.push(c); loop { - c = self.lookahead_char()?; + c = try_nt!(token, self.lookahead_char()); if decimal_digit_char!(c) { self.skip_char(c); @@ -757,60 +785,29 @@ impl<'a, R: CharRead> Lexer<'a, R> { } let n = parse_float_lossy(&token)?; - Ok(Token::Literal(Literal::from(float_alloc!( + Ok(NumberToken::Number(Number::Float(float_alloc!( n, self.machine_st.arena )))) } else { - return self.vacate_with_float(token); + return self.vacate_with_float(token).map(NumberToken::Number); } } else { let n = parse_float_lossy(&token)?; - Ok(Token::Literal(Literal::from(float_alloc!( + Ok(NumberToken::Number(Number::Float(float_alloc!( n, self.machine_st.arena )))) } } else { self.return_char('.'); - - token - .parse::() - .map(|n| Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena))) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) - }) + self.parse_integer(&token).map(NumberToken::Number) } } else if token.starts_with('0') && token.len() == 1 { if c == 'x' { self.hexadecimal_constant(c).or_else(|e| { if let ParserError::ParseBigInt(..) = e { - token - .parse::() - .map(|n| { - Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena)) - }) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| { - ParserError::ParseBigInt(self.line_num, self.col_num) - }) - }) + self.parse_integer(&token).map(NumberToken::Number) } else { Err(e) } @@ -818,24 +815,7 @@ impl<'a, R: CharRead> Lexer<'a, R> { } else if c == 'o' { self.octal_constant(c).or_else(|e| { if let ParserError::ParseBigInt(..) = e { - token - .parse::() - .map(|n| { - Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena)) - }) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| { - ParserError::ParseBigInt(self.line_num, self.col_num) - }) - }) + self.parse_integer(&token).map(NumberToken::Number) } else { Err(e) } @@ -843,24 +823,7 @@ impl<'a, R: CharRead> Lexer<'a, R> { } else if c == 'b' { self.binary_constant(c).or_else(|e| { if let ParserError::ParseBigInt(..) = e { - token - .parse::() - .map(|n| { - Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena)) - }) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| { - ParserError::ParseBigInt(self.line_num, self.col_num) - }) - }) + self.parse_integer(&token).map(NumberToken::Number) } else { Err(e) } @@ -877,14 +840,14 @@ impl<'a, R: CharRead> Lexer<'a, R> { self.skip_char(c); self.return_char('\''); - return Ok(Token::Literal(Literal::Fixnum(Fixnum::build_with(0)))); + return Ok(NumberToken::Number(Number::Fixnum(Fixnum::build_with(0)))); } else { self.return_char('\\'); } } self.get_single_quoted_char() - .map(|c| Token::Literal(Literal::Fixnum(Fixnum::build_with(c as i64)))) + .map(|c| NumberToken::Number(Number::Fixnum(Fixnum::build_with(c as i64)))) .or_else(|err| { match err { ParserError::UnexpectedChar('\'', ..) => {} @@ -892,57 +855,13 @@ impl<'a, R: CharRead> Lexer<'a, R> { } self.return_char(c); - - token - .parse::() - .map(|n| { - Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena)) - }) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| { - ParserError::ParseBigInt(self.line_num, self.col_num) - }) - }) + self.parse_integer(&token).map(NumberToken::Number) }) } else { - token - .parse::() - .map(|n| Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena))) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) - }) + self.parse_integer(&token).map(NumberToken::Number) } } else { - token - .parse::() - .map(|n| Token::Literal(fixnum!(Literal, n, &mut self.machine_st.arena))) - .or_else(|_| { - token - .parse::() - .map(|n| { - Token::Literal(Literal::Integer(arena_alloc!( - n, - &mut self.machine_st.arena - ))) - }) - .map_err(|_| ParserError::ParseBigInt(self.line_num, self.col_num)) - }) + self.parse_integer(&token).map(NumberToken::Number) } } @@ -1001,6 +920,29 @@ impl<'a, R: CharRead> Lexer<'a, R> { } } + pub fn next_number_token(&mut self) -> Result { + self.scan_for_layout()?; + let c = self.lookahead_char()?; + + if !decimal_digit_char!(c) { + return self.name_token(c); + } + + match self.number_token(c) { + Ok(NumberToken::Partial(token_string)) => match self.parse_integer(&token_string) { + Ok(n) => Ok(Token::Literal(n.to_literal())), + Err(_) => { + let n = parse_float_lossy(&token_string)?; + Ok(Token::Literal(Literal::Float( + float_alloc!(n, self.machine_st.arena).as_offset(), + ))) + } + }, + Ok(NumberToken::Number(n)) => return Ok(Token::Literal(n.to_literal())), + Err(e) => return Err(e), + } + } + pub fn next_token(&mut self) -> Result { let layout_inserted = self.scan_for_layout()?; let cr = self.lookahead_char(); @@ -1053,7 +995,10 @@ impl<'a, R: CharRead> Lexer<'a, R> { } if decimal_digit_char!(c) { - return self.number_token(c); + return self.number_token(c).and_then(|nt| match nt.to_token() { + Some(token) => Ok(token), + None => Err(ParserError::unexpected_eof()), + }); } if c == ']' { From 778068129f9ff1e6cd3eb69cb44b3337c5aab52e Mon Sep 17 00:00:00 2001 From: Mark Thom Date: Mon, 7 Jul 2025 21:28:07 -0700 Subject: [PATCH 09/10] don't skip '.' char in parse_number_from_string (#2997) --- src/machine/system_calls.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/machine/system_calls.rs b/src/machine/system_calls.rs index f14645e7..cc39cba5 100644 --- a/src/machine/system_calls.rs +++ b/src/machine/system_calls.rs @@ -968,9 +968,6 @@ impl MachineState { break; } - Ok('.') => { - lexer.skip_char('.'); - } Ok(c) => { let (line_num, col_num) = (lexer.line_num, lexer.col_num); From a69303dc9315592bedc3374ca5a92e13f2be0b8b Mon Sep 17 00:00:00 2001 From: Mark Thom Date: Mon, 7 Jul 2025 21:30:04 -0700 Subject: [PATCH 10/10] throw syntax error after parsing infinite floats (#2998) --- src/parser/ast.rs | 5 +++++ src/parser/parser.rs | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/parser/ast.rs b/src/parser/ast.rs index 06c1ab9e..9a400c36 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -432,6 +432,7 @@ pub enum ParserError { BackQuotedString(usize, usize), IO(IOError), IncompleteReduction(usize, usize), + InfiniteFloat(usize, usize), InvalidSingleQuotedCharacter(char), LexicalError(lexical::Error), MissingQuote(usize, usize), @@ -447,6 +448,7 @@ impl ParserError { match self { &ParserError::BackQuotedString(line_num, col_num) | &ParserError::IncompleteReduction(line_num, col_num) + | &ParserError::InfiniteFloat(line_num, col_num) | &ParserError::MissingQuote(line_num, col_num) | &ParserError::NonPrologChar(line_num, col_num) | &ParserError::ParseBigInt(line_num, col_num) @@ -463,6 +465,9 @@ impl ParserError { ParserError::InvalidSingleQuotedCharacter(..) => { atom!("invalid_single_quoted_character") } + ParserError::InfiniteFloat(..) => { + atom!("infinite_float") + } ParserError::IO(e) if e.kind() == ErrorKind::UnexpectedEof => { atom!("unexpected_end_of_file") } diff --git a/src/parser/parser.rs b/src/parser/parser.rs index eee22294..dd7b007b 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -976,11 +976,17 @@ impl<'a, R: CharRead> Parser<'a, R> { Token::Literal(Literal::Rational(n)) => { self.negate_number(n, negate_rat_rc, |r, _| Literal::Rational(r)) } - Token::Literal(Literal::Float(n)) => self.negate_number( + Token::Literal(Literal::Float(n)) if F64Ptr::from_offset(n).is_infinite() => { + return Err(ParserError::InfiniteFloat( + self.lexer.line_num, + self.lexer.col_num, + )); + } + Token::Literal(Literal::Float(n)) => self.negate_number( **n.as_ptr(), |n, _| -n, |n, arena| Literal::from(float_alloc!(n, arena)), - ), + ), Token::Literal(c) => { let atomized = atomize_constant(&self.lexer.machine_st.atom_tbl, c);