Benchmark improvements
* CI: Clean offline; display cleaned bytes; one artifact upload * Add missing csv bench to iai benchmarks * Generate flamegraphs for benchmarks * Validate benchmark results separately
This commit is contained in:
infogulch
7
.github/actions/setup-rust/cleanup.sh
vendored
7
.github/actions/setup-rust/cleanup.sh
vendored
@@ -3,8 +3,11 @@ set -e
|
||||
|
||||
echo Cleanup workspace build artifacts and extra target output
|
||||
|
||||
# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects
|
||||
cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`
|
||||
# clean just the direct members of the current workspace, use cargo metadata to generalize to all rust projects
|
||||
cargo clean -p `cargo metadata --no-deps --offline --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`
|
||||
|
||||
# remove directories in /target/ that are not named `debug` or `release`
|
||||
before=`du -s target | awk '{print $1}'`
|
||||
find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \;
|
||||
after=`du -s target | awk '{print $1}'`
|
||||
echo Deleted $(($before - $after)) bytes from target directory
|
||||
|
||||
18
.github/workflows/ci.yml
vendored
18
.github/workflows/ci.yml
vendored
@@ -161,21 +161,19 @@ jobs:
|
||||
fail_on: nothing
|
||||
comment_mode: off
|
||||
|
||||
- run: cargo build --bench run_iai --release
|
||||
- run: cargo build --all-targets --release
|
||||
- run: cargo test --bench setup --release
|
||||
- run: cargo bench --bench run_iai
|
||||
- run: cargo bench --bench run_criterion
|
||||
- run: cargo bench --bench run_criterion -- --profile-time 60
|
||||
|
||||
- name: Publish criterion benchmark results
|
||||
- name: Publish benchmark results
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: bench-criterion-results
|
||||
path: target/criterion/**/*
|
||||
|
||||
- name: Publish iai benchmark results
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: bench-iai-results
|
||||
path: target/iai/scryer-prolog/run_iai/bench_group/*
|
||||
name: benchmark-results
|
||||
path: |
|
||||
target/criterion/*
|
||||
target/iai/*
|
||||
|
||||
# Publish binaries when building for a tag
|
||||
release:
|
||||
|
||||
216
Cargo.lock
generated
216
Cargo.lock
generated
@@ -17,6 +17,19 @@ version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.0.2"
|
||||
@@ -59,6 +72,12 @@ version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
|
||||
|
||||
[[package]]
|
||||
name = "assert_cmd"
|
||||
version = "1.0.8"
|
||||
@@ -234,6 +253,12 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.4.3"
|
||||
@@ -380,6 +405,15 @@ version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
|
||||
|
||||
[[package]]
|
||||
name = "cpp_demangle"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpu-time"
|
||||
version = "1.0.0"
|
||||
@@ -620,6 +654,15 @@ dependencies = [
|
||||
"num-order",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "debugid"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
|
||||
dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_deref"
|
||||
version = "1.1.1"
|
||||
@@ -711,6 +754,12 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.3"
|
||||
@@ -759,6 +808,18 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "findshlibs"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
@@ -975,7 +1036,7 @@ dependencies = [
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"http",
|
||||
"indexmap",
|
||||
"indexmap 1.9.3",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
@@ -1219,6 +1280,34 @@ dependencies = [
|
||||
"hashbrown 0.12.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.14.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inferno"
|
||||
version = "0.11.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c50453ec3a6555fad17b1cd1a80d16af5bc7cb35094f64e429fd46549018c6a3"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"indexmap 2.0.1",
|
||||
"is-terminal",
|
||||
"itoa",
|
||||
"log",
|
||||
"num-format",
|
||||
"once_cell",
|
||||
"quick-xml",
|
||||
"rgb",
|
||||
"str_stack",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.12"
|
||||
@@ -1300,7 +1389,7 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"arrayvec 0.5.2",
|
||||
"bitflags 1.3.2",
|
||||
"cfg-if",
|
||||
"ryu",
|
||||
@@ -1414,6 +1503,15 @@ version = "2.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43a5a03cefb0d953ec0be133036f14e109412fa594edc2f77227249db66cc3ed"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.0"
|
||||
@@ -1582,6 +1680,16 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-format"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3"
|
||||
dependencies = [
|
||||
"arrayvec 0.7.4",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-modular"
|
||||
version = "0.6.1"
|
||||
@@ -1898,6 +2006,28 @@ dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pprof"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"cfg-if",
|
||||
"criterion",
|
||||
"findshlibs",
|
||||
"inferno",
|
||||
"libc",
|
||||
"log",
|
||||
"nix 0.26.4",
|
||||
"once_cell",
|
||||
"parking_lot 0.12.1",
|
||||
"smallvec",
|
||||
"symbolic-demangle",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
@@ -1976,6 +2106,15 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.26.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.33"
|
||||
@@ -2158,6 +2297,15 @@ dependencies = [
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rgb"
|
||||
version = "0.8.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.16.20"
|
||||
@@ -2342,7 +2490,7 @@ dependencies = [
|
||||
"git-version",
|
||||
"hostname",
|
||||
"iai-callgrind",
|
||||
"indexmap",
|
||||
"indexmap 1.9.3",
|
||||
"lazy_static",
|
||||
"lexical",
|
||||
"libc",
|
||||
@@ -2354,6 +2502,7 @@ dependencies = [
|
||||
"num-order",
|
||||
"ordered-float",
|
||||
"phf 0.9.0",
|
||||
"pprof",
|
||||
"predicates-core",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -2637,6 +2786,12 @@ version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
@@ -2649,6 +2804,12 @@ version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0"
|
||||
|
||||
[[package]]
|
||||
name = "str_stack"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.8.7"
|
||||
@@ -2706,6 +2867,29 @@ version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
|
||||
|
||||
[[package]]
|
||||
name = "symbolic-common"
|
||||
version = "12.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "405af7bd5edd866cef462e22ef73f11cf9bf506c9d62824fef8364eb69d4d4ad"
|
||||
dependencies = [
|
||||
"debugid",
|
||||
"memmap2",
|
||||
"stable_deref_trait",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "symbolic-demangle"
|
||||
version = "12.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bcd041ccfb77d9c70639efcd5b804b508ac7a273e9224d227379e225625daf9"
|
||||
dependencies = [
|
||||
"cpp_demangle",
|
||||
"rustc-demangle",
|
||||
"symbolic-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
@@ -3047,6 +3231,12 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
@@ -3352,3 +3542,23 @@ name = "xmlparser"
|
||||
version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
]
|
||||
|
||||
11
Cargo.toml
11
Cargo.toml
@@ -115,9 +115,20 @@ serial_test = "2.0.0"
|
||||
iai-callgrind = "0.7.3"
|
||||
criterion = "0.5.1"
|
||||
|
||||
[target.'cfg(not(target_os = "windows"))'.dev-dependencies]
|
||||
pprof = { version = "0.13.0", features = ["criterion", "flamegraph"] }
|
||||
|
||||
[patch.crates-io]
|
||||
modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }
|
||||
|
||||
[profile.bench]
|
||||
lto = true
|
||||
opt-level = 3
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
opt-level = 3
|
||||
|
||||
[[bench]]
|
||||
name = "run_criterion"
|
||||
harness = false
|
||||
|
||||
@@ -4,16 +4,16 @@ The `benches` directory contains benchmarks that test scryer-prolog performance.
|
||||
|
||||
Benchmarks are run via two harnesses:
|
||||
|
||||
* `criterion` - criterion performs statistical analysis of benchmark runs and is
|
||||
great for benchmarking locally.
|
||||
* `iai-callgrind` - this runs the benchmark with callgrind, which is able to
|
||||
precisely track the number of instructions executed during the run. This is
|
||||
especially helpful in a public CI runner context where neighboring VMs can
|
||||
cause a very high wall time variance. This means that it doesn't track wall
|
||||
time which is what we really care about, but it is a good tradeoff for CI
|
||||
where tracking runtime is unreliable.
|
||||
* `criterion` - criterion performs statistical analysis of benchmark runs and is
|
||||
great for benchmarking locally.
|
||||
cause a very high wall time variance. While instructions executed is only
|
||||
correlated with the desired metric (wall time), this is a good tradeoff for CI
|
||||
where that metric is unreliable.
|
||||
|
||||
Run them using the following commands:
|
||||
Run benchmarks with the following commands:
|
||||
|
||||
```
|
||||
cargo bench --bench run_criterion
|
||||
@@ -21,6 +21,9 @@ cargo bench --bench run_criterion
|
||||
# run a particular criterion benchmark
|
||||
cargo bench --bench run_criterion -- <benchmark_name>
|
||||
|
||||
# run in profiling mode which outputs flamegraphs. Set profile time in seconds:
|
||||
cargo bench --bench run_criterion -- --profile-time <time>
|
||||
|
||||
# to run iai, you need valgrind installed and to install iai-callgrind-runner
|
||||
# at the same version as is in Cargo.toml:
|
||||
cargo install iai-callgrind-runner --version 0.7.3
|
||||
@@ -29,7 +32,7 @@ cargo bench --bench run_iai
|
||||
```
|
||||
|
||||
For consistency, both runners -- `run_iai.rs` and `run_criterion.rs` -- import
|
||||
the same setup code from `benches.rs`.
|
||||
the same setup code from `setup.rs`.
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -50,45 +53,43 @@ This design is meant to suppoort defining lots of benchmarks.
|
||||
|
||||
To add a new benchmark:
|
||||
|
||||
* Add a new file `benches/[module].pl` that contains setup code. Import
|
||||
* Add a new file `benches/[module].pl` that contains setup prolog code. Import
|
||||
libraries, define predicates, etc.
|
||||
* Add a new section in `setup.rs::benches()` that refers to it and add some
|
||||
benchmarks.
|
||||
* Add a new section in `setup.rs::prolog_benchmarks()` that refers to to the
|
||||
file and write a query to be benchmarked.
|
||||
* If the query mutates the machine, then use `Strategy::Fresh` so the criterion
|
||||
benchmark will recreate a new machine for each benchmark run, otherwise use
|
||||
`Strategy::Reuse` which has lower overhead. (This is not used by the iai
|
||||
benchmark because it only runs once anyway.)
|
||||
|
||||
Some tips:
|
||||
|
||||
* The goal of benchmarking is to know if a library or engine change improved
|
||||
performance or not.
|
||||
* Once a benchmark is defined and named, avoid changing it's definition. In
|
||||
general, if a benchmark needs to change to be more useful, give the new
|
||||
definition a new name. This will prevent charts from showing wild changes in
|
||||
* Once a benchmark is defined and named, avoid changing it's definition. If a
|
||||
benchmark needs to change to be more useful, give the new definition a new
|
||||
name instead. This will prevent charts from showing wild changes in
|
||||
performance just because the definition changed (see previous).
|
||||
* Aim for queries to execute in less than 0.5s realtime. Longer runtimes make it
|
||||
easier for humans to see big differences, but benchmarks either run 10x slower
|
||||
(iai) or execute repeatedly to attain statistical significance (criterion) and
|
||||
in both cases queries that take longer become cumbersome to run.
|
||||
in both cases benchmarking queries that take longer than about 0.5s are
|
||||
cumbersome to run.
|
||||
* Consider that the library runtime actually parses the text output of the top
|
||||
level. So don't use custom outputs or it will fail to parse. Also keep the
|
||||
output small so it doesn't just benchmark the ouput parsing code.
|
||||
* DO test the output of the benchmark run, we don't want to count broken
|
||||
benchmarks.
|
||||
* Because a query may run against the same machine multiple times, don't
|
||||
permanently mutate the state of the engine with the query since that will
|
||||
taint subsequent runs. (Benchmarking assertz et al is desirable, but will
|
||||
require some adjustments to how the machine is set up for runs.)
|
||||
|
||||
## CI
|
||||
|
||||
Both benchmark harnesses are run in `.github/workflows/ci.yaml` in the `report`
|
||||
job, and the results are published as build artifacts.
|
||||
|
||||
A future action may consume the build artifacts and publish a report using the
|
||||
results.
|
||||
|
||||
## Todo
|
||||
|
||||
- [ ] Currently, the execution time to load a module is not benchmarked. It
|
||||
would be nice to have at least one benchmark for loading a module (probably a
|
||||
big one).
|
||||
- [ ] Write a new action that consumes the test and benchmark results and plots
|
||||
them over time and publishes a report (github pages?).
|
||||
- [ ] Write a new action that downloads the test and benchmark results
|
||||
artifacts, plots them over time, and publishes a report to github pages.
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
use pprof::criterion::{Output, PProfProfiler};
|
||||
|
||||
mod setup;
|
||||
|
||||
fn bench_criterion(c: &mut Criterion) {
|
||||
@@ -13,9 +16,21 @@ fn bench_criterion(c: &mut Criterion) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
fn config() -> Criterion {
|
||||
Criterion::default()
|
||||
.sample_size(20)
|
||||
.with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn config() -> Criterion {
|
||||
Criterion::default().sample_size(20)
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
name = bench_group;
|
||||
config = Criterion::default().sample_size(10);
|
||||
name = benches;
|
||||
config = config();
|
||||
targets = bench_criterion
|
||||
);
|
||||
criterion_main!(bench_group);
|
||||
criterion_main!(benches);
|
||||
|
||||
@@ -1,21 +1,18 @@
|
||||
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
|
||||
use scryer_prolog::machine::parsed_results::QueryResolution;
|
||||
|
||||
mod setup;
|
||||
|
||||
#[library_benchmark]
|
||||
#[bench::normal(setup::prolog_benches()["count_edges_short"].setup())]
|
||||
fn bench_edges(mut run: impl FnMut()) {
|
||||
run();
|
||||
}
|
||||
|
||||
#[library_benchmark]
|
||||
#[bench::normal(setup::prolog_benches()["numlist_short"].setup())]
|
||||
fn bench_numlist(mut run: impl FnMut()) {
|
||||
run();
|
||||
#[bench::count_edges(setup::prolog_benches()["count_edges"].setup())]
|
||||
#[bench::numlist(setup::prolog_benches()["numlist"].setup())]
|
||||
#[bench::csv_codename(setup::prolog_benches()["csv_codename"].setup())]
|
||||
fn bench(mut run: impl FnMut() -> QueryResolution) -> QueryResolution {
|
||||
run()
|
||||
}
|
||||
|
||||
library_benchmark_group!(
|
||||
name = bench_group;
|
||||
benchmarks = bench_edges, bench_numlist
|
||||
name = benches;
|
||||
benchmarks = bench
|
||||
);
|
||||
main!(library_benchmark_groups = bench_group);
|
||||
main!(library_benchmark_groups = benches);
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::{collections::BTreeMap, fs, path::Path};
|
||||
|
||||
use maplit::btreemap;
|
||||
use scryer_prolog::machine::{
|
||||
parsed_results::{QueryMatch, QueryResolution, Value},
|
||||
parsed_results::{QueryResolution, Value},
|
||||
Machine,
|
||||
};
|
||||
|
||||
@@ -10,20 +10,13 @@ pub fn prolog_benches() -> BTreeMap<&'static str, PrologBenchmark> {
|
||||
[
|
||||
(
|
||||
"count_edges", // name of the benchmark
|
||||
"benches/edges.pl", // name of the prolog module file to load
|
||||
"independent_set_count(aa, Count).", // query to benchmark in the context of the loaded module
|
||||
Strategy::Reuse,
|
||||
btreemap! { "Count" => Value::try_from("211954906".to_string()).unwrap(), }, // list of expected bindings
|
||||
),
|
||||
(
|
||||
"count_edges_short",
|
||||
"benches/edges.pl", // use the same file in multiple benchmarks
|
||||
"independent_set_count(ky, Count).", // consider making the query adjustable to tune the run time to ~0.1s
|
||||
"benches/edges.pl", // name of the prolog module file to load. use the same file in multiple benchmarks
|
||||
"independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module. consider making the query adjustable to tune the run time to ~0.1s
|
||||
Strategy::Reuse,
|
||||
btreemap! { "Count" => Value::try_from("2869176".to_string()).unwrap() },
|
||||
),
|
||||
(
|
||||
"numlist_short",
|
||||
"numlist",
|
||||
"benches/numlist.pl",
|
||||
"run_numlist(1000000, Head).",
|
||||
Strategy::Reuse,
|
||||
@@ -67,27 +60,40 @@ pub struct PrologBenchmark {
|
||||
}
|
||||
|
||||
impl PrologBenchmark {
|
||||
pub fn setup(&self) -> impl FnMut() {
|
||||
pub fn make_machine(&self) -> Machine {
|
||||
let program = fs::read_to_string(self.filename).unwrap();
|
||||
let module_name = Path::new(self.filename)
|
||||
.file_stem()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap();
|
||||
|
||||
let mut machine = Machine::new_lib();
|
||||
machine.load_module_string(module_name, program);
|
||||
machine
|
||||
}
|
||||
|
||||
let benchmark_name = self.name;
|
||||
pub fn setup(&self) -> impl FnMut() -> QueryResolution {
|
||||
let mut machine = self.make_machine();
|
||||
let query = self.query;
|
||||
let expected = QueryResolution::Matches(vec![QueryMatch::from(self.bindings.clone())]);
|
||||
|
||||
move || {
|
||||
use criterion::black_box;
|
||||
let result = black_box(machine.run_query(black_box(query.to_string())));
|
||||
match result {
|
||||
Ok(r) => assert_eq!(&r, &expected),
|
||||
Err(e) => panic!("benchmark {} failed with: {}", benchmark_name, e),
|
||||
}
|
||||
black_box(machine.run_query(black_box(query.to_string()))).unwrap()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
#[test]
|
||||
fn validate_benchmarks() {
|
||||
use super::prolog_benches;
|
||||
use scryer_prolog::machine::parsed_results::QueryResolution;
|
||||
|
||||
use scryer_prolog::machine::parsed_results::QueryMatch;
|
||||
for (_, r) in prolog_benches() {
|
||||
let mut machine = r.make_machine();
|
||||
let result = machine.run_query(r.query.to_string()).unwrap();
|
||||
let expected = QueryResolution::Matches(vec![QueryMatch::from(r.bindings.clone())]);
|
||||
assert_eq!(result, expected, "validating benchmark {}", r.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user