restore prolog_parser test files (#782)
This commit is contained in:
Mark Thom
265
crates/prolog_parser/Cargo.lock
generated
Normal file
265
crates/prolog_parser/Cargo.lock
generated
Normal file
@@ -0,0 +1,265 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
|
||||
dependencies = [
|
||||
"nodrop",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||
|
||||
[[package]]
|
||||
name = "az"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d84e1d907bfc5795a6addb95ef8666141ee73c8f2f5250ff2a46bf4e4f4aec8a"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||
|
||||
[[package]]
|
||||
name = "gmp-mpfr-sys"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a57fdb339d49833021b1fded600ed240ae907e33909d5511a61dff884df7f16e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical"
|
||||
version = "2.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e0d09e60c187a6d0a3fa418aec8587c6a4ae9de872f6126f2134f319b5ed10d"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"lexical-core",
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-core"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304bccb228c4b020f3a4835d247df0a02a7c4686098d4167762cfbbe4c5cb14"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"cfg-if",
|
||||
"rustc_version",
|
||||
"ryu",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.85"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ccac4b00700875e6a07c6cde370d44d32fa01c5a65cdd2fca6858c479d28bb3"
|
||||
|
||||
[[package]]
|
||||
name = "nodrop"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-integer",
|
||||
"num-traits 0.2.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-traits 0.2.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits 0.2.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rug-adapter"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7470b6acf85abce0771203112db4181d03f7b8a6be49f0e842a78030192f8a58"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-rational",
|
||||
"num-traits 0.2.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.1.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
|
||||
dependencies = [
|
||||
"num-traits 0.2.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7eb5259643245d3f292c7a146b2df53bba24d7eab159410e648eb73dc164669d"
|
||||
dependencies = [
|
||||
"num-traits 0.1.43",
|
||||
"unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prolog_parser_rebis"
|
||||
version = "0.8.68"
|
||||
dependencies = [
|
||||
"lexical",
|
||||
"num-rug-adapter",
|
||||
"ordered-float",
|
||||
"rug",
|
||||
"unicode_reader",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rug"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e538d00da450a8e48aac7e6322e67b2dc86ec71a1feeac0e3954c4f07f01bc45"
|
||||
dependencies = [
|
||||
"az",
|
||||
"gmp-mpfr-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
|
||||
dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
|
||||
dependencies = [
|
||||
"semver-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver-parser"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
|
||||
|
||||
[[package]]
|
||||
name = "unicode_reader"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b639121690b27acd92c97ed2b52c5e5e8d3d39482e943b4559695cef62f771a"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unreachable"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
|
||||
dependencies = [
|
||||
"void",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "void"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
43
crates/prolog_parser/tests/bom.rs
Normal file
43
crates/prolog_parser/tests/bom.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
extern crate prolog_parser_rebis;
|
||||
|
||||
use prolog_parser_rebis::ast::*;
|
||||
use prolog_parser_rebis::lexer::{Lexer, Token};
|
||||
use prolog_parser_rebis::tabled_rc::TabledData;
|
||||
|
||||
use std::rc::Rc;
|
||||
|
||||
#[test]
|
||||
fn valid_token() {
|
||||
let stream = parsing_stream("valid text".as_bytes());
|
||||
assert!(stream.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_stream() {
|
||||
let bytes: &[u8] = &[];
|
||||
assert!(parsing_stream(bytes).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skip_utf8_bom() {
|
||||
let atom_tbl = TabledData::new(Rc::new("my_module".to_string()));
|
||||
let flags = MachineFlags::default();
|
||||
let bytes: &[u8] = &[0xEF, 0xBB, 0xBF, '4' as u8, '\n' as u8];
|
||||
let mut stream = parsing_stream(bytes).expect("valid stream");
|
||||
let mut lexer = Lexer::new(atom_tbl, flags, &mut stream);
|
||||
match lexer.next_token() {
|
||||
Ok(Token::Constant(Constant::Fixnum(4))) => (),
|
||||
_ => assert!(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_utf16_bom() {
|
||||
let bytes: &[u8] = &[0xFF, 0xFE, 'a' as u8, '\n' as u8];
|
||||
let stream = parsing_stream(bytes);
|
||||
match stream {
|
||||
Err(ParserError::Utf8Error(0, 0)) => (),
|
||||
_ => assert!(false)
|
||||
}
|
||||
}
|
||||
|
||||
114
crates/prolog_parser/tests/parse_tokens.rs
Normal file
114
crates/prolog_parser/tests/parse_tokens.rs
Normal file
@@ -0,0 +1,114 @@
|
||||
extern crate prolog_parser_rebis;
|
||||
|
||||
use prolog_parser_rebis::ast::*;
|
||||
use prolog_parser_rebis::lexer::{Lexer, Token};
|
||||
use prolog_parser_rebis::tabled_rc::TabledData;
|
||||
|
||||
use std::rc::Rc;
|
||||
|
||||
fn read_all_tokens(text: &str) -> Result<Vec<Token>, ParserError> {
|
||||
let atom_tbl = TabledData::new(Rc::new("my_module".to_string()));
|
||||
let flags = MachineFlags::default();
|
||||
let mut stream = parsing_stream(text.as_bytes())?;
|
||||
let mut lexer = Lexer::new(atom_tbl, flags, &mut stream);
|
||||
|
||||
let mut tokens = Vec::new();
|
||||
while !lexer.eof()? {
|
||||
let token = lexer.next_token()?;
|
||||
tokens.push(token);
|
||||
}
|
||||
Ok(tokens)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_multiline_comment() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens("/**/ 4\n")?;
|
||||
assert_eq!(tokens, [Token::Constant(Constant::Fixnum(4))]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_char_multiline_comment() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens("/* █╗╚═══╝ © */ 4\n")?;
|
||||
assert_eq!(tokens, [Token::Constant(Constant::Fixnum(4))]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn simple_char() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens("'a'\n")?;
|
||||
assert_eq!(tokens, [Token::Constant(Constant::Char('a'))]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_with_meta_seq() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens(r#"'\\' '\'' '\"' '\`' "#)?; // use literal string so \ are escaped
|
||||
assert_eq!(
|
||||
tokens,
|
||||
[
|
||||
Token::Constant(Constant::Char('\\')),
|
||||
Token::Constant(Constant::Char('\'')),
|
||||
Token::Constant(Constant::Char('"')),
|
||||
Token::Constant(Constant::Char('`'))
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_with_control_seq() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens(r"'\a' '\b' '\r' '\f' '\t' '\n' '\v' ")?;
|
||||
assert_eq!(
|
||||
tokens,
|
||||
[
|
||||
Token::Constant(Constant::Char('\u{07}')),
|
||||
Token::Constant(Constant::Char('\u{08}')),
|
||||
Token::Constant(Constant::Char('\r')),
|
||||
Token::Constant(Constant::Char('\u{0c}')),
|
||||
Token::Constant(Constant::Char('\t')),
|
||||
Token::Constant(Constant::Char('\n')),
|
||||
Token::Constant(Constant::Char('\u{0b}')),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_with_octseq() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens(r"'\60433\' ")?;
|
||||
assert_eq!(tokens, [Token::Constant(Constant::Char('愛'))]); // Japanese character
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_with_octseq_0() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens(r"'\0\' ")?;
|
||||
assert_eq!(tokens, [Token::Constant(Constant::Char('\u{0000}'))]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_with_hexseq() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens(r"'\x2124\' ")?;
|
||||
assert_eq!(tokens, [Token::Constant(Constant::Char('ℤ'))]); // Z math symbol
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn char_with_hexseq_invalid() {
|
||||
assert!(read_all_tokens(r"'\x\' ").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty() -> Result<(), ParserError> {
|
||||
let tokens = read_all_tokens("")?;
|
||||
assert!(tokens.is_empty());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comment_then_eof() -> Result<(), ParserError> {
|
||||
assert!(read_all_tokens("% only a comment").is_err());
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user