rishi/scryer-prolog
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add benchmarks using library interface

Browse Source
This commit is contained in:
infogulch
2023-11-11 11:22:02 -06:00
committed by Joe Taber
parent 1656d9d7ad
commit 8570f119c0
12 changed files with 727 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/actions/setup-rust/action.yml vendored
View File

@@ -48,7 +48,5 @@ runs:
# Must be placed after actions/cache so its post step runs first.
- uses: pyTooling/Actions/with-post-step@v0.4.6
with:
main: "true"
post: |
echo Post job cleanup: bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`"
bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`"
main: bash ./.github/actions/setup-rust/cleanup.sh
post: bash ./.github/actions/setup-rust/cleanup.sh

10
.github/actions/setup-rust/cleanup.sh vendored Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
echo Cleanup workspace build artifacts and extra target output
# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects
cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`
# remove directories in /target/ that are not named `debug` or `release`
find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \;

18
.github/workflows/ci.yml vendored
View File

@@ -139,6 +139,8 @@ jobs:
cache-context: report
- run: |
cargo install cargo2junit --force
cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'`
sudo apt install valgrind -y
- name: Test and report
run: |
@@ -156,6 +158,22 @@ jobs:
fail_on: nothing
comment_mode: off
- run: cargo build --bench run_iai --release
- run: cargo bench --bench run_iai
- run: cargo bench --bench run_criterion
- name: Publish criterion benchmark results
uses: actions/upload-artifact@v3
with:
name: bench-criterion-results
path: target/criterion/**/*
- name: Publish iai benchmark results
uses: actions/upload-artifact@v3
with:
name: bench-iai-results
path: target/iai/scryer-prolog/run_iai/bench_group/*
# Publish binaries when building for a tag
release:
runs-on: ubuntu-20.04

296
Cargo.lock generated
View File

@@ -41,6 +41,18 @@ dependencies = [
"libc",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstyle"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
[[package]]
name = "arrayvec"
version = "0.5.2"
@@ -100,6 +112,15 @@ version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bit-set"
version = "0.5.3"
@@ -225,6 +246,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.0.83"
@@ -254,6 +281,58 @@ dependencies = [
"windows-targets",
]
[[package]]
name = "ciborium"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656"
[[package]]
name = "ciborium-ll"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "4.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663"
dependencies = [
"anstyle",
"clap_lex",
]
[[package]]
name = "clap_lex"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
[[package]]
name = "clipboard-win"
version = "4.5.0"
@@ -320,6 +399,75 @@ dependencies = [
"libc",
]
[[package]]
name = "criterion"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
dependencies = [
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"is-terminal",
"itertools",
"num-traits",
"once_cell",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossterm"
version = "0.20.0"
@@ -834,6 +982,12 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -990,6 +1144,38 @@ dependencies = [
"tokio-native-tls",
]
[[package]]
name = "iai-callgrind"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2679de583a9f5232b45b1f3a31b5e2f739bccfee9b7902488aca8f8c2c5482d1"
dependencies = [
"bincode",
"iai-callgrind-macros",
"iai-callgrind-runner",
]
[[package]]
name = "iai-callgrind-macros"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af5af66b85e350097b8c0f6329c6347d3323d010443475741c29a1a167f116fb"
dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"syn 2.0.37",
]
[[package]]
name = "iai-callgrind-runner"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12cc1093f263249d7c541c53849dac74f4ae756cd31e3cac1a12204efcec68bd"
dependencies = [
"serde",
]
[[package]]
name = "iana-time-zone"
version = "0.1.57"
@@ -1048,6 +1234,17 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
[[package]]
name = "is-terminal"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
dependencies = [
"hermit-abi",
"rustix",
"windows-sys",
]
[[package]]
name = "itertools"
version = "0.10.5"
@@ -1217,6 +1414,15 @@ version = "2.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
[[package]]
name = "memoffset"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -1425,6 +1631,12 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "oorandom"
version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "opaque-debug"
version = "0.2.3"
@@ -1658,6 +1870,34 @@ version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "plotters"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609"
[[package]]
name = "plotters-svg"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab"
dependencies = [
"plotters-backend",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@@ -1697,6 +1937,30 @@ dependencies = [
"termtree",
]
[[package]]
name = "proc-macro-error"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn 1.0.109",
"version_check",
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check",
]
[[package]]
name = "proc-macro-hack"
version = "0.5.20+deprecated"
@@ -1767,6 +2031,26 @@ dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
@@ -2044,6 +2328,7 @@ dependencies = [
"console_error_panic_hook",
"console_log",
"cpu-time",
"criterion",
"crossterm",
"crrl",
"ctrlc",
@@ -2056,6 +2341,7 @@ dependencies = [
"getrandom",
"git-version",
"hostname",
"iai-callgrind",
"indexmap",
"lazy_static",
"lexical",
@@ -2498,6 +2784,16 @@ dependencies = [
"syn 2.0.37",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tinyvec"
version = "1.6.0"

26
Cargo.toml
View File

@@ -85,7 +85,13 @@ tokio = { version = "1.28.2", features = ["full"] }
[target.'cfg(target_arch = "wasm32")'.dependencies]
getrandom = { version = "0.2.10", features = ["js"] }
tokio = { version = "1.28.2", features = ["sync", "macros", "io-util", "rt", "time"] }
tokio = { version = "1.28.2", features = [
"sync",
"macros",
"io-util",
"rt",
"time",
] }
[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies]
console_error_panic_hook = "0.1"
@@ -93,11 +99,7 @@ console_log = "1.0"
wasm-bindgen = "0.2.87"
wasm-bindgen-futures = "0.4"
serde-wasm-bindgen = "0.5"
web-sys = { version = "0.3", features = [
"Document",
"Window",
"Element",
]}
web-sys = { version = "0.3", features = ["Document", "Window", "Element"] }
[target.'cfg(target_os = "wasi")'.dependencies]
ring-wasi = { version = "0.16.25" }
@@ -110,6 +112,16 @@ assert_cmd = "1.0.3"
predicates-core = "1.0.2"
maplit = "1.0.2"
serial_test = "2.0.0"
iai-callgrind = "0.7.3"
criterion = "0.5.1"
[patch.crates-io]
modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }
modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }
[[bench]]
name = "run_criterion"
harness = false
[[bench]]
name = "run_iai"
harness = false

95
benches/README.md Normal file
View File

@@ -0,0 +1,95 @@
# About benches
The `benches` directory contains benchmarks that test scryer-prolog performance.
Benchmarks are run via two harnesses:
* `iai-callgrind` - this runs the benchmark with callgrind, which is able to
precisely track the number of instructions executed during the run. This is
especially helpful in a public CI runner context where neighboring VMs can
cause a very high wall time variance. This means that it doesn't track wall
time which is what we really care about, but it is a good tradeoff for CI
where tracking runtime is unreliable.
* `criterion` - criterion performs statistical analysis of benchmark runs and is
great for benchmarking locally.
Run them using the following commands:
```
cargo bench --bench run_criterion
# to run iai, you need valgrind installed and to install iai-callgrind-runner
# at the same version as is in Cargo.toml:
cargo install iai-callgrind-runner --version 0.7.3
cargo bench --bench run_iai
```
For consistency, both runners -- `run_iai.rs` and `run_criterion.rs` -- import
the same setup code from `benches.rs`.
## Setup
`setup.rs` contains the setup code for the actual benchmarks, which are run
using the `Benches` struct. `fn benches()` at the top of the file is where the
benchmarks are defined.
Benchmarks are organized around running queries against one prolog module file.
Before any runs start, `Benches::new()` reads the module files and initializes a
new `scryer_prolog::machine::Machine` for each file; multiple queries can be
declared to be benchmarked in the context of that module/machine instance.
Each benchmark measurement is done by running a query against the machine. In
the case of criterion each query is run many times, in the case of iai it's run
once.
## Adding benchmarks
This design is meant to suppoort defining lots of benchmarks.
To add a new benchmark:
* Add a new file `benches/[module].pl` that contains setup code. Import
libraries, define predicates, etc.
* Add a new section in `setup.rs::benches()` that refers to it and add some
benchmarks.
Some tips:
* The goal of benchmarking is to know if a library or engine change improved
performance or not.
* Once a benchmark is defined and named, don't change it's definition. If a
benchmark needs to change to be more useful, give the new definition a new
name. This will prevent charts from showing wild changes in performance just
because the definition changed (see previous).
* Aim for queries to execute in about 0.1-0.5s realtime. Longer runtimes make it
easier for humans to see big differences, but benchmarks either run 10x slower
(iai) or execute repeatedly to attain statistical significance (criterion) and
in both cases queries that take 5+ seconds quickly become unweildly.
* Consider that the library runtime actually parses the text output of the top
level. So keep the output small and don't use custom outputs or it will fail
to parse.
* DO test the output of the benchmark run, we don't want to count broken
benchmarks.
* Because a query may run against the same machine multiple times, don't
permanently mutate the state of the engine with the query since that will
taint subsequent runs. (Benchmarking assertz et al is desirable, but will
require some adjustments to how the machine is set up for runs.)
## CI
Both benchmark harnesses are run in `.github/workflows/ci.yaml` in the `report`
job, and the results are published as build artifacts.
A future action may consume the build artifacts and publish a report using the
results.
## Todo
- [ ] Currently, the execution time to load a module is not benchmarked. It
would be nice to have at least one benchmark for loading a module (probably a
big one).
- [ ] Adjust the benchmark execution strategy to allow queries to modify the
engine state (`assertz` etc).
- [ ] Write a new action that consumes the test and benchmark results and plots
them over time and publishes a report (github pages?).

130
benches/edges.pl Normal file
View File

@@ -0,0 +1,130 @@
:- use_module(library(clpb)).
:- use_module(library(assoc)).
:- use_module(library(lists)).
:- use_module(library(pairs)).
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Contiguous United States and DC as they appear in SGB:
http://www-cs-faculty.stanford.edu/~uno/sgb.html
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
edge(al, fl).
edge(al, ga).
edge(al, ms).
edge(al, tn).
edge(ar, la).
edge(ar, mo).
edge(ar, ms).
edge(ar, ok).
edge(ar, tn).
edge(ar, tx).
edge(az, ca).
edge(az, nm).
edge(az, nv).
edge(az, ut).
edge(ca, nv).
edge(ca, or).
edge(co, ks).
edge(co, ne).
edge(co, nm).
edge(co, ok).
edge(co, ut).
edge(co, wy).
edge(ct, ma).
edge(ct, ny).
edge(ct, ri).
edge(dc, md).
edge(dc, va).
edge(de, md).
edge(de, nj).
edge(de, pa).
edge(fl, ga).
edge(ga, nc).
edge(ga, sc).
edge(ga, tn).
edge(ia, il).
edge(ia, mn).
edge(ia, mo).
edge(ia, ne).
edge(ia, sd).
edge(ia, wi).
edge(id, mt).
edge(id, nv).
edge(id, or).
edge(id, ut).
edge(id, wa).
edge(id, wy).
edge(il, in).
edge(il, ky).
edge(il, mo).
edge(il, wi).
edge(in, ky).
edge(in, mi).
edge(in, oh).
edge(ks, mo).
edge(ks, ne).
edge(ks, ok).
edge(ky, mo).
edge(ky, oh).
edge(ky, tn).
edge(ky, va).
edge(ky, wv).
edge(la, ms).
edge(la, tx).
edge(ma, nh).
edge(ma, ny).
edge(ma, ri).
edge(ma, vt).
edge(md, pa).
edge(md, va).
edge(md, wv).
edge(me, nh).
edge(mi, oh).
edge(mi, wi).
edge(mn, nd).
edge(mn, sd).
edge(mn, wi).
edge(mo, ne).
edge(mo, ok).
edge(mo, tn).
edge(ms, tn).
edge(mt, nd).
edge(mt, sd).
edge(mt, wy).
edge(nc, sc).
edge(nc, tn).
edge(nc, va).
edge(nd, sd).
edge(ne, sd).
edge(ne, wy).
edge(nh, vt).
edge(nj, ny).
edge(nj, pa).
edge(nm, ok).
edge(nm, tx).
edge(nv, or).
edge(nv, ut).
edge(ny, pa).
edge(ny, vt).
edge(oh, pa).
edge(oh, wv).
edge(ok, tx).
edge(or, wa).
edge(pa, wv).
edge(sd, wy).
edge(tn, va).
edge(ut, wy).
edge(va, wv).
independent_set(G, *(NBs)) :-
findall(U-V, (edge(U, V),G@<U), Edges),
setof(U, V^(member(U-V, Edges);member(V-U, Edges)), Nodes),
pairs_keys_values(Pairs, Nodes, _),
list_to_assoc(Pairs, Assoc),
maplist(not_both(Assoc), Edges, NBs).
not_both(Assoc, U-V, ~BU + ~BV) :-
get_assoc(U, Assoc, BU),
get_assoc(V, Assoc, BV).
independent_set_count(G, Count) :- independent_set(G, Sat), sat_count(Sat, Count).

2
benches/numlist.pl Normal file
View File

@@ -0,0 +1,2 @@
:- use_module(library(between)).
run_numlist(Upper, Head) :- numlist(1, Upper, L), L = [Head|_].

14
benches/run_criterion.rs Normal file
View File

@@ -0,0 +1,14 @@
use criterion::{criterion_group, criterion_main, Criterion};
mod setup;
fn bench_criterion(c: &mut Criterion) {
setup::benches().run_all_criterion(c);
}
criterion_group!(
name = bench_group;
config = Criterion::default().sample_size(10);
targets = bench_criterion
);
criterion_main!(bench_group);

21
benches/run_iai.rs Normal file
View File

@@ -0,0 +1,21 @@
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
mod setup;
#[library_benchmark]
#[bench::normal(setup::benches())]
fn bench_edges(mut b: setup::Benches) {
b.run_once("count_edges_short");
}
#[library_benchmark]
#[bench::normal(setup::benches())]
fn bench_numlist(mut b: setup::Benches) {
b.run_once("numlist_short");
}
library_benchmark_group!(
name = bench_group;
benchmarks = bench_edges, bench_numlist
);
main!(library_benchmark_groups = bench_group);

116
benches/setup.rs Normal file
View File

@@ -0,0 +1,116 @@
use std::{collections::BTreeMap, fs, path::Path};
use criterion::{black_box, Criterion};
use maplit::btreemap;
use scryer_prolog::machine::{
parsed_results::{QueryMatch, QueryResolution, Value},
Machine,
};
pub fn benches() -> Benches {
Benches::new(&[
(
"benches/edges.pl", // name of the prolog module file to load
&[
(
"count_edges_short", // name of the benchmark
"independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module
btreemap! { "Count".to_string() => Value::try_from("2869176".to_string()).unwrap() }, // List of expected bindings
),
(
"count_edges", // multiple benchmark queries can be defined per module
"independent_set_count(aa, Count).", // consider making the query adjustable to tune the runtime
btreemap! { "Count".to_string() => Value::try_from("211954906".to_string()).unwrap(), },
),
],
),
(
"benches/numlist.pl",
&[(
"numlist_short",
"run_numlist(1000000, Head).",
btreemap! { "Head".to_string() => Value::try_from("1".to_string()).unwrap()},
)],
),
])
}
pub struct Benches {
machines: Vec<Machine>,
runs: BTreeMap<String, Run>,
}
pub struct Run {
machine_idx: usize,
name: &'static str,
query: &'static str,
bindings: BTreeMap<String, Value>,
}
// Required for using a mutex. It doesn't actually send anything across threads,
// and this is just a benchmark, so it Should Be Fine(tm). ¯\_(ツ)_/¯
unsafe impl Send for Benches {}
impl Benches {
#[allow(clippy::type_complexity)]
pub fn new(
benches: &[(
&'static str,
&[(&'static str, &'static str, BTreeMap<String, Value>)],
)],
) -> Self {
let mut machines = vec![];
let mut runs = BTreeMap::new();
for b in benches {
let content = fs::read_to_string(b.0).unwrap();
let name = Path::new(b.0).file_stem().unwrap().to_str().unwrap();
let mut machine = Machine::new_lib();
machine.load_module_string(name, content);
machines.push(machine);
let idx = machines.len() - 1;
runs.extend(b.1.iter().cloned().map(|r| {
(
r.0.to_string(),
Run {
machine_idx: idx,
name: r.0,
query: r.1,
bindings: r.2,
},
)
}));
}
Benches { machines, runs }
}
#[allow(dead_code)]
pub fn run_all_criterion(&mut self, c: &mut Criterion) {
for (_, runner) in self.runs.iter() {
let machine = &mut self.machines[runner.machine_idx];
c.bench_function(runner.name, |b| {
b.iter(|| {
Self::run(machine, runner);
})
});
}
}
#[allow(dead_code)]
pub fn run_once(&mut self, name: &str) {
let runner = &self.runs[name];
let machine = &mut self.machines[runner.machine_idx];
Self::run(machine, runner);
}
fn run(machine: &mut Machine, runner: &Run) {
assert_eq!(
black_box(machine.run_query(black_box(runner.query.to_string()))),
Ok(QueryResolution::Matches(vec![QueryMatch::from(
runner.bindings.clone()
)]))
);
}
}

14
src/machine/lib_machine.rs
View File

@@ -60,7 +60,7 @@ impl Machine {
}
pub fn run_query(&mut self, query: String) -> QueryResult {
println!("Query: {}", query);
// println!("Query: {}", query);
// Parse the query so we can analyze and then call the term
let mut parser = Parser::new(
Stream::from_owned_string(query, &mut self.machine_st.arena),
@@ -192,7 +192,7 @@ impl Machine {
let outputter = printer.print();
let output: String = outputter.result();
println!("Result: {} = {}", var_key.to_string(), output);
// println!("Result: {} = {}", var_key.to_string(), output);
bindings.insert(var_key.to_string(), Value::try_from(output).expect("asdfs"));
}
@@ -445,10 +445,7 @@ mod tests {
}
// Check if the block is a query
if block.starts_with("query") {
// Extract the query from the block
let query = &block[5..];
if let Some(query) = block.strip_prefix("query") {
i += 1;
println!("query #{}: {}", i, query);
// Parse and execute the query
@@ -458,10 +455,7 @@ mod tests {
// Print the result
println!("{:?}", result);
} else if block.starts_with("consult") {
// Extract the code from the block
let code = &block[7..];
} else if let Some(code) = block.strip_prefix("consult") {
println!("load code: {}", code);
// Load the code into the machine