From e7fd408548a9fbbc19586a6b6cdb0020082cac75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Tue, 23 Jul 2024 00:01:29 +0200 Subject: [PATCH 1/7] use include_str! for the build in libraries instead of "copy-paste-ing" the prolog source into rust string literals in the generated files --- build/main.rs | 68 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/build/main.rs b/build/main.rs index b7a66f03..1f026de3 100644 --- a/build/main.rs +++ b/build/main.rs @@ -5,23 +5,18 @@ use instructions_template::generate_instructions_rs; use static_string_indexing::index_static_strings; use std::env; -use std::fs; use std::fs::File; use std::io::Write; use std::path::Path; +use std::path::PathBuf; use std::process::{Command, Stdio}; -fn find_prolog_files( - libraries: &mut File, - path_prefix: &str, - const_prefix: &str, - current_dir: &Path, -) -> Vec<(String, String)> { - let mut constants = vec![]; +fn find_prolog_files(path_prefix: &str, current_dir: &Path) -> Vec<(String, PathBuf)> { + let mut libraries = vec![]; let entries = match current_dir.read_dir() { Ok(entries) => entries, - Err(_) => return constants, + Err(_) => return libraries, }; for entry in entries.filter_map(Result::ok).map(|e| e.path()) { @@ -29,27 +24,21 @@ fn find_prolog_files( if let Some(file_name) = entry.file_name() { let file_name = file_name.to_str().unwrap(); let new_path_prefix = format!("{path_prefix}{file_name}/"); - let new_const_prefix = format!("{const_prefix}_{}", file_name.to_uppercase()); - let new_consts = - find_prolog_files(libraries, &new_path_prefix, &new_const_prefix, &entry); - constants.extend(new_consts); + let new_libs = find_prolog_files(&new_path_prefix, &entry); + libraries.extend(new_libs); } } else if entry.is_file() { let ext = std::ffi::OsStr::new("pl"); if entry.extension() == Some(ext) { - let contain = String::from_utf8(fs::read(&entry).unwrap()).unwrap(); let name = entry.file_stem().unwrap().to_str().unwrap(); let lib_name = format!("{path_prefix}{name}"); - let const_name = format!("{const_prefix}_{}", name.to_uppercase()); - writeln!(libraries, "const {const_name}: &str = {contain:?};").unwrap(); - - constants.push((lib_name, const_name)); + libraries.push((lib_name, entry)); } } } - constants + libraries } fn main() { @@ -67,32 +56,59 @@ fn main() { let dest_path = Path::new(&out_dir).join("libraries.rs"); let mut libraries = File::create(dest_path).unwrap(); - let lib_path = Path::new("src/lib"); + let lib_path = Path::new("src").join("lib"); writeln!( libraries, "\ -use indexmap::IndexMap;\ +use indexmap::IndexMap; +\ " ) .unwrap(); - let constants = find_prolog_files(&mut libraries, "", "LIB", lib_path); + let constants = find_prolog_files("", &lib_path); writeln!( libraries, "\ std::thread_local!{{ static LIBRARIES: IndexMap<&'static str, &'static str> = {{ - let mut m = IndexMap::new();" + let mut m = IndexMap::new(); +\ + " ) .unwrap(); - for (name, constant) in constants { + let out_dir = std::env::var("OUT_DIR").unwrap(); + let out_dir_path: &Path = out_dir.as_ref(); + let manifest_dir = &std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let manifest_dir_path: &Path = manifest_dir.as_ref(); + + let prefix: PathBuf = if let Ok(diff) = out_dir_path.strip_prefix(manifest_dir_path) { + let mut path = PathBuf::from("."); + for comp in diff.components() { + match comp { + std::path::Component::Normal(_) => path.push(".."), + std::path::Component::CurDir => (), + std::path::Component::Prefix(_) + | std::path::Component::RootDir + | std::path::Component::ParentDir => { + path = manifest_dir_path.to_path_buf(); + break; + } + } + } + path + } else { + manifest_dir_path.to_path_buf() + }; + + for (name, lib_path) in constants { + let path: PathBuf = prefix.join(lib_path); writeln!( libraries, - "\ - m.insert(\"{name}\",{constant});" + " m.insert(\"{name}\", include_str!({path:?}));" ) .unwrap(); } From 1eb6a883f6534bc6c1bce948943ce2fd1c0de532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Tue, 23 Jul 2024 00:01:58 +0200 Subject: [PATCH 2/7] run rustfmt --- src/http.rs | 2 +- src/machine/streams.rs | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/http.rs b/src/http.rs index f301ef64..bd6ec33b 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,4 +1,4 @@ -use bytes::{Bytes, buf::Reader}; +use bytes::{buf::Reader, Bytes}; use std::sync::{Arc, Condvar, Mutex}; use warp::http; diff --git a/src/machine/streams.rs b/src/machine/streams.rs index 2a0b59e6..670f3204 100644 --- a/src/machine/streams.rs +++ b/src/machine/streams.rs @@ -15,6 +15,8 @@ use crate::types::*; use bytes::Buf; pub use scryer_modular_bitfield::prelude::*; +#[cfg(feature = "http")] +use bytes::{buf::Reader as BufReader, Bytes}; use std::cmp::Ordering; use std::error::Error; use std::fmt; @@ -22,8 +24,6 @@ use std::fmt::Debug; use std::fs::{File, OpenOptions}; use std::hash::Hash; use std::io; -#[cfg(feature = "http")] -use bytes::{buf::Reader as BufReader, Bytes}; use std::io::{Cursor, ErrorKind, Read, Seek, SeekFrom, Write}; use std::net::{Shutdown, TcpStream}; use std::ops::{Deref, DerefMut}; @@ -1136,13 +1136,19 @@ impl Stream { } } } - Stream::HttpRead(stream_layout) => { - if stream_layout.stream.get_ref().body_reader.get_ref().has_remaining() { - AtEndOfStream::Not - } else { - AtEndOfStream::Past - } - } + Stream::HttpRead(stream_layout) => { + if stream_layout + .stream + .get_ref() + .body_reader + .get_ref() + .has_remaining() + { + AtEndOfStream::Not + } else { + AtEndOfStream::Past + } + } _ => AtEndOfStream::Not, } } From 6b50ec328cd02508494620d766afd4270683b690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Thu, 25 Jul 2024 18:53:36 +0200 Subject: [PATCH 3/7] move the creation of the LIBRARIES IndexMap out of the generated file - only leave the filling of the map in the generated code --- build/main.rs | 37 +++---------------------------------- src/machine/mod.rs | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/build/main.rs b/build/main.rs index 1f026de3..a43b9e86 100644 --- a/build/main.rs +++ b/build/main.rs @@ -58,28 +58,8 @@ fn main() { let mut libraries = File::create(dest_path).unwrap(); let lib_path = Path::new("src").join("lib"); - writeln!( - libraries, - "\ -use indexmap::IndexMap; -\ - " - ) - .unwrap(); - let constants = find_prolog_files("", &lib_path); - writeln!( - libraries, - "\ -std::thread_local!{{ - static LIBRARIES: IndexMap<&'static str, &'static str> = {{ - let mut m = IndexMap::new(); -\ - " - ) - .unwrap(); - let out_dir = std::env::var("OUT_DIR").unwrap(); let out_dir_path: &Path = out_dir.as_ref(); let manifest_dir = &std::env::var("CARGO_MANIFEST_DIR").unwrap(); @@ -104,23 +84,12 @@ std::thread_local!{{ manifest_dir_path.to_path_buf() }; + writeln!(libraries, "{{").unwrap(); for (name, lib_path) in constants { let path: PathBuf = prefix.join(lib_path); - writeln!( - libraries, - " m.insert(\"{name}\", include_str!({path:?}));" - ) - .unwrap(); + writeln!(libraries, "m.insert(\"{name}\", include_str!({path:?}));").unwrap(); } - - writeln!( - libraries, - " - m - }}; -}}" - ) - .unwrap(); + writeln!(libraries, "}}").unwrap(); let instructions_path = Path::new(&out_dir).join("instructions.rs"); let mut instructions_file = File::create(&instructions_path).unwrap(); diff --git a/src/machine/mod.rs b/src/machine/mod.rs index 33199edd..12170e39 100644 --- a/src/machine/mod.rs +++ b/src/machine/mod.rs @@ -119,7 +119,17 @@ fn current_dir() -> PathBuf { } mod libraries { - include!(concat!(env!("OUT_DIR"), "/libraries.rs")); + use indexmap::IndexMap; + + std::thread_local! { + static LIBRARIES: IndexMap<&'static str, &'static str> = { + let mut m = IndexMap::new(); + + include!(concat!(env!("OUT_DIR"), "/libraries.rs")); + + m + } + } pub(crate) fn contains(name: &str) -> bool { LIBRARIES.with(|libs| libs.contains_key(name)) @@ -128,16 +138,6 @@ mod libraries { pub(crate) fn get(name: &str) -> Option<&'static str> { LIBRARIES.with(|libs| libs.get(name).copied()) } - - #[cfg(test)] - std::thread_local! { - #[allow(dead_code)] - static LIBRARIES2 : IndexMap<&'static str, &'static str> = { - let mut m = IndexMap::new(); - m.insert("test", "test2"); - m - }; - } } pub static BREAK_FROM_DISPATCH_LOOP_LOC: usize = 0; From 0da313eb8495c02d4c2d216f1074ec1e0a062550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Thu, 25 Jul 2024 18:58:29 +0200 Subject: [PATCH 4/7] use a OnceLock static for LIBRARIES instead of thread_local! static --- src/machine/mod.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/machine/mod.rs b/src/machine/mod.rs index 12170e39..1726ae5b 100644 --- a/src/machine/mod.rs +++ b/src/machine/mod.rs @@ -120,23 +120,25 @@ fn current_dir() -> PathBuf { mod libraries { use indexmap::IndexMap; + use std::sync::OnceLock; - std::thread_local! { - static LIBRARIES: IndexMap<&'static str, &'static str> = { + fn libraries() -> &'static IndexMap<&'static str, &'static str> { + static LIBRARIES: OnceLock> = OnceLock::new(); + LIBRARIES.get_or_init(|| { let mut m = IndexMap::new(); include!(concat!(env!("OUT_DIR"), "/libraries.rs")); m - } + }) } pub(crate) fn contains(name: &str) -> bool { - LIBRARIES.with(|libs| libs.contains_key(name)) + libraries().contains_key(name) } pub(crate) fn get(name: &str) -> Option<&'static str> { - LIBRARIES.with(|libs| libs.get(name).copied()) + libraries().get(name).copied() } } From 3ad4b05a97642810b3975c03d6adff14b331146c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Thu, 25 Jul 2024 23:09:17 +0200 Subject: [PATCH 5/7] add a features for gating things we can use once 1.80 is msrv --- Cargo.toml | 1 + src/machine/mod.rs | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 67c3e60d..180015ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ hostname = ["dep:hostname"] tls = ["dep:native-tls"] http = ["dep:warp", "dep:reqwest"] crypto-full = [] +"rust-version-1.80" = [] [build-dependencies] indexmap = "1.0.2" diff --git a/src/machine/mod.rs b/src/machine/mod.rs index 1726ae5b..9110ae9c 100644 --- a/src/machine/mod.rs +++ b/src/machine/mod.rs @@ -118,6 +118,7 @@ fn current_dir() -> PathBuf { } } +#[cfg(not(feature = "rust-version-1.80"))] mod libraries { use indexmap::IndexMap; use std::sync::OnceLock; @@ -142,6 +143,28 @@ mod libraries { } } +#[cfg(feature = "rust-version-1.80")] +mod libraries { + use indexmap::IndexMap; + use std::sync::LazyLock; + + static LIBRARIES: LazyLock> = OnceLock::new(|| { + let mut m = IndexMap::new(); + + include!(concat!(env!("OUT_DIR"), "/libraries.rs")); + + m + }); + + pub(crate) fn contains(name: &str) -> bool { + LIBRARIES.contains_key(name) + } + + pub(crate) fn get(name: &str) -> Option<&'static str> { + LIBRARIES.get(name).copied() + } +} + pub static BREAK_FROM_DISPATCH_LOOP_LOC: usize = 0; pub static INSTALL_VERIFY_ATTR_INTERRUPT: usize = 1; pub static VERIFY_ATTR_INTERRUPT_LOC: usize = 2; From c29b76143c5097fff435e7a457e4d843ec8fab1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bennet=20Ble=C3=9Fmann?= Date: Fri, 26 Jul 2024 00:13:49 +0200 Subject: [PATCH 6/7] fix copy-past error --- src/machine/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/machine/mod.rs b/src/machine/mod.rs index 9110ae9c..cfba5666 100644 --- a/src/machine/mod.rs +++ b/src/machine/mod.rs @@ -148,7 +148,7 @@ mod libraries { use indexmap::IndexMap; use std::sync::LazyLock; - static LIBRARIES: LazyLock> = OnceLock::new(|| { + static LIBRARIES: LazyLock> = LazyLock::new(|| { let mut m = IndexMap::new(); include!(concat!(env!("OUT_DIR"), "/libraries.rs")); From e310f9683ea771cec624d7a325ca3eba646f1f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Arroyo=20Calle?= Date: Mon, 8 Jul 2024 16:12:10 +0200 Subject: [PATCH 7/7] Fix build on WASM after #2442 (cherry-pick of 3acf4c637b6f8b17bb0230edeaaae0507d9030d1 from PR mthom/scryer-prolog#2446) --- src/machine/streams.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/machine/streams.rs b/src/machine/streams.rs index 670f3204..2a49cc0c 100644 --- a/src/machine/streams.rs +++ b/src/machine/streams.rs @@ -12,11 +12,10 @@ use crate::machine::machine_indices::*; use crate::machine::machine_state::*; use crate::types::*; -use bytes::Buf; pub use scryer_modular_bitfield::prelude::*; #[cfg(feature = "http")] -use bytes::{buf::Reader as BufReader, Bytes}; +use bytes::{buf::Reader as BufReader, Buf, Bytes}; use std::cmp::Ordering; use std::error::Error; use std::fmt; @@ -1136,6 +1135,7 @@ impl Stream { } } } + #[cfg(feature = "http")] Stream::HttpRead(stream_layout) => { if stream_layout .stream