Add benchmarks using library interface
This commit is contained in:
infogulch
6
.github/actions/setup-rust/action.yml
vendored
6
.github/actions/setup-rust/action.yml
vendored
@@ -48,7 +48,5 @@ runs:
|
||||
# Must be placed after actions/cache so its post step runs first.
|
||||
- uses: pyTooling/Actions/with-post-step@v0.4.6
|
||||
with:
|
||||
main: "true"
|
||||
post: |
|
||||
echo Post job cleanup: bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`"
|
||||
bash -c "cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(\" \")|.[0]]|join(\" \")'`"
|
||||
main: bash ./.github/actions/setup-rust/cleanup.sh
|
||||
post: bash ./.github/actions/setup-rust/cleanup.sh
|
||||
|
||||
10
.github/actions/setup-rust/cleanup.sh
vendored
Executable file
10
.github/actions/setup-rust/cleanup.sh
vendored
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
echo Cleanup workspace build artifacts and extra target output
|
||||
|
||||
# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects
|
||||
cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`
|
||||
|
||||
# remove directories in /target/ that are not named `debug` or `release`
|
||||
find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \;
|
||||
18
.github/workflows/ci.yml
vendored
18
.github/workflows/ci.yml
vendored
@@ -139,6 +139,8 @@ jobs:
|
||||
cache-context: report
|
||||
- run: |
|
||||
cargo install cargo2junit --force
|
||||
cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'`
|
||||
sudo apt install valgrind -y
|
||||
|
||||
- name: Test and report
|
||||
run: |
|
||||
@@ -156,6 +158,22 @@ jobs:
|
||||
fail_on: nothing
|
||||
comment_mode: off
|
||||
|
||||
- run: cargo build --bench run_iai --release
|
||||
- run: cargo bench --bench run_iai
|
||||
- run: cargo bench --bench run_criterion
|
||||
|
||||
- name: Publish criterion benchmark results
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: bench-criterion-results
|
||||
path: target/criterion/**/*
|
||||
|
||||
- name: Publish iai benchmark results
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: bench-iai-results
|
||||
path: target/iai/scryer-prolog/run_iai/bench_group/*
|
||||
|
||||
# Publish binaries when building for a tag
|
||||
release:
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
296
Cargo.lock
generated
296
Cargo.lock
generated
@@ -41,6 +41,18 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anes"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.5.2"
|
||||
@@ -100,6 +112,15 @@ version = "0.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.5.3"
|
||||
@@ -225,6 +246,12 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
|
||||
|
||||
[[package]]
|
||||
name = "cast"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.83"
|
||||
@@ -254,6 +281,58 @@ dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"ciborium-ll",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-io"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656"
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-ll"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
|
||||
|
||||
[[package]]
|
||||
name = "clipboard-win"
|
||||
version = "4.5.0"
|
||||
@@ -320,6 +399,75 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
|
||||
dependencies = [
|
||||
"anes",
|
||||
"cast",
|
||||
"ciborium",
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"is-terminal",
|
||||
"itertools",
|
||||
"num-traits",
|
||||
"once_cell",
|
||||
"oorandom",
|
||||
"plotters",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"tinytemplate",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion-plot"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm"
|
||||
version = "0.20.0"
|
||||
@@ -834,6 +982,12 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "1.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
@@ -990,6 +1144,38 @@ dependencies = [
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iai-callgrind"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2679de583a9f5232b45b1f3a31b5e2f739bccfee9b7902488aca8f8c2c5482d1"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"iai-callgrind-macros",
|
||||
"iai-callgrind-runner",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iai-callgrind-macros"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af5af66b85e350097b8c0f6329c6347d3323d010443475741c29a1a167f116fb"
|
||||
dependencies = [
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.37",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iai-callgrind-runner"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12cc1093f263249d7c541c53849dac74f4ae756cd31e3cac1a12204efcec68bd"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.57"
|
||||
@@ -1048,6 +1234,17 @@ version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
@@ -1217,6 +1414,15 @@ version = "2.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@@ -1425,6 +1631,12 @@ version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.2.3"
|
||||
@@ -1658,6 +1870,34 @@ version = "0.3.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"plotters-backend",
|
||||
"plotters-svg",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters-backend"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609"
|
||||
|
||||
[[package]]
|
||||
name = "plotters-svg"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab"
|
||||
dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
@@ -1697,6 +1937,30 @@ dependencies = [
|
||||
"termtree",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
|
||||
dependencies = [
|
||||
"proc-macro-error-attr",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error-attr"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-hack"
|
||||
version = "0.5.20+deprecated"
|
||||
@@ -1767,6 +2031,26 @@ dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
@@ -2044,6 +2328,7 @@ dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"console_log",
|
||||
"cpu-time",
|
||||
"criterion",
|
||||
"crossterm",
|
||||
"crrl",
|
||||
"ctrlc",
|
||||
@@ -2056,6 +2341,7 @@ dependencies = [
|
||||
"getrandom",
|
||||
"git-version",
|
||||
"hostname",
|
||||
"iai-callgrind",
|
||||
"indexmap",
|
||||
"lazy_static",
|
||||
"lexical",
|
||||
@@ -2498,6 +2784,16 @@ dependencies = [
|
||||
"syn 2.0.37",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.6.0"
|
||||
|
||||
24
Cargo.toml
24
Cargo.toml
@@ -85,7 +85,13 @@ tokio = { version = "1.28.2", features = ["full"] }
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies]
|
||||
getrandom = { version = "0.2.10", features = ["js"] }
|
||||
tokio = { version = "1.28.2", features = ["sync", "macros", "io-util", "rt", "time"] }
|
||||
tokio = { version = "1.28.2", features = [
|
||||
"sync",
|
||||
"macros",
|
||||
"io-util",
|
||||
"rt",
|
||||
"time",
|
||||
] }
|
||||
|
||||
[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies]
|
||||
console_error_panic_hook = "0.1"
|
||||
@@ -93,11 +99,7 @@ console_log = "1.0"
|
||||
wasm-bindgen = "0.2.87"
|
||||
wasm-bindgen-futures = "0.4"
|
||||
serde-wasm-bindgen = "0.5"
|
||||
web-sys = { version = "0.3", features = [
|
||||
"Document",
|
||||
"Window",
|
||||
"Element",
|
||||
]}
|
||||
web-sys = { version = "0.3", features = ["Document", "Window", "Element"] }
|
||||
|
||||
[target.'cfg(target_os = "wasi")'.dependencies]
|
||||
ring-wasi = { version = "0.16.25" }
|
||||
@@ -110,6 +112,16 @@ assert_cmd = "1.0.3"
|
||||
predicates-core = "1.0.2"
|
||||
maplit = "1.0.2"
|
||||
serial_test = "2.0.0"
|
||||
iai-callgrind = "0.7.3"
|
||||
criterion = "0.5.1"
|
||||
|
||||
[patch.crates-io]
|
||||
modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }
|
||||
|
||||
[[bench]]
|
||||
name = "run_criterion"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "run_iai"
|
||||
harness = false
|
||||
|
||||
95
benches/README.md
Normal file
95
benches/README.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# About benches
|
||||
|
||||
The `benches` directory contains benchmarks that test scryer-prolog performance.
|
||||
|
||||
Benchmarks are run via two harnesses:
|
||||
|
||||
* `iai-callgrind` - this runs the benchmark with callgrind, which is able to
|
||||
precisely track the number of instructions executed during the run. This is
|
||||
especially helpful in a public CI runner context where neighboring VMs can
|
||||
cause a very high wall time variance. This means that it doesn't track wall
|
||||
time which is what we really care about, but it is a good tradeoff for CI
|
||||
where tracking runtime is unreliable.
|
||||
* `criterion` - criterion performs statistical analysis of benchmark runs and is
|
||||
great for benchmarking locally.
|
||||
|
||||
Run them using the following commands:
|
||||
|
||||
```
|
||||
cargo bench --bench run_criterion
|
||||
|
||||
# to run iai, you need valgrind installed and to install iai-callgrind-runner
|
||||
# at the same version as is in Cargo.toml:
|
||||
cargo install iai-callgrind-runner --version 0.7.3
|
||||
|
||||
cargo bench --bench run_iai
|
||||
```
|
||||
|
||||
For consistency, both runners -- `run_iai.rs` and `run_criterion.rs` -- import
|
||||
the same setup code from `benches.rs`.
|
||||
|
||||
## Setup
|
||||
|
||||
`setup.rs` contains the setup code for the actual benchmarks, which are run
|
||||
using the `Benches` struct. `fn benches()` at the top of the file is where the
|
||||
benchmarks are defined.
|
||||
|
||||
Benchmarks are organized around running queries against one prolog module file.
|
||||
Before any runs start, `Benches::new()` reads the module files and initializes a
|
||||
new `scryer_prolog::machine::Machine` for each file; multiple queries can be
|
||||
declared to be benchmarked in the context of that module/machine instance.
|
||||
|
||||
Each benchmark measurement is done by running a query against the machine. In
|
||||
the case of criterion each query is run many times, in the case of iai it's run
|
||||
once.
|
||||
|
||||
## Adding benchmarks
|
||||
|
||||
This design is meant to suppoort defining lots of benchmarks.
|
||||
|
||||
To add a new benchmark:
|
||||
|
||||
* Add a new file `benches/[module].pl` that contains setup code. Import
|
||||
libraries, define predicates, etc.
|
||||
* Add a new section in `setup.rs::benches()` that refers to it and add some
|
||||
benchmarks.
|
||||
|
||||
Some tips:
|
||||
|
||||
* The goal of benchmarking is to know if a library or engine change improved
|
||||
performance or not.
|
||||
* Once a benchmark is defined and named, don't change it's definition. If a
|
||||
benchmark needs to change to be more useful, give the new definition a new
|
||||
name. This will prevent charts from showing wild changes in performance just
|
||||
because the definition changed (see previous).
|
||||
* Aim for queries to execute in about 0.1-0.5s realtime. Longer runtimes make it
|
||||
easier for humans to see big differences, but benchmarks either run 10x slower
|
||||
(iai) or execute repeatedly to attain statistical significance (criterion) and
|
||||
in both cases queries that take 5+ seconds quickly become unweildly.
|
||||
* Consider that the library runtime actually parses the text output of the top
|
||||
level. So keep the output small and don't use custom outputs or it will fail
|
||||
to parse.
|
||||
* DO test the output of the benchmark run, we don't want to count broken
|
||||
benchmarks.
|
||||
* Because a query may run against the same machine multiple times, don't
|
||||
permanently mutate the state of the engine with the query since that will
|
||||
taint subsequent runs. (Benchmarking assertz et al is desirable, but will
|
||||
require some adjustments to how the machine is set up for runs.)
|
||||
|
||||
## CI
|
||||
|
||||
Both benchmark harnesses are run in `.github/workflows/ci.yaml` in the `report`
|
||||
job, and the results are published as build artifacts.
|
||||
|
||||
A future action may consume the build artifacts and publish a report using the
|
||||
results.
|
||||
|
||||
## Todo
|
||||
|
||||
- [ ] Currently, the execution time to load a module is not benchmarked. It
|
||||
would be nice to have at least one benchmark for loading a module (probably a
|
||||
big one).
|
||||
- [ ] Adjust the benchmark execution strategy to allow queries to modify the
|
||||
engine state (`assertz` etc).
|
||||
- [ ] Write a new action that consumes the test and benchmark results and plots
|
||||
them over time and publishes a report (github pages?).
|
||||
130
benches/edges.pl
Normal file
130
benches/edges.pl
Normal file
@@ -0,0 +1,130 @@
|
||||
:- use_module(library(clpb)).
|
||||
:- use_module(library(assoc)).
|
||||
:- use_module(library(lists)).
|
||||
:- use_module(library(pairs)).
|
||||
|
||||
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
Contiguous United States and DC as they appear in SGB:
|
||||
http://www-cs-faculty.stanford.edu/~uno/sgb.html
|
||||
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
|
||||
|
||||
edge(al, fl).
|
||||
edge(al, ga).
|
||||
edge(al, ms).
|
||||
edge(al, tn).
|
||||
edge(ar, la).
|
||||
edge(ar, mo).
|
||||
edge(ar, ms).
|
||||
edge(ar, ok).
|
||||
edge(ar, tn).
|
||||
edge(ar, tx).
|
||||
edge(az, ca).
|
||||
edge(az, nm).
|
||||
edge(az, nv).
|
||||
edge(az, ut).
|
||||
edge(ca, nv).
|
||||
edge(ca, or).
|
||||
edge(co, ks).
|
||||
edge(co, ne).
|
||||
edge(co, nm).
|
||||
edge(co, ok).
|
||||
edge(co, ut).
|
||||
edge(co, wy).
|
||||
edge(ct, ma).
|
||||
edge(ct, ny).
|
||||
edge(ct, ri).
|
||||
edge(dc, md).
|
||||
edge(dc, va).
|
||||
edge(de, md).
|
||||
edge(de, nj).
|
||||
edge(de, pa).
|
||||
edge(fl, ga).
|
||||
edge(ga, nc).
|
||||
edge(ga, sc).
|
||||
edge(ga, tn).
|
||||
edge(ia, il).
|
||||
edge(ia, mn).
|
||||
edge(ia, mo).
|
||||
edge(ia, ne).
|
||||
edge(ia, sd).
|
||||
edge(ia, wi).
|
||||
edge(id, mt).
|
||||
edge(id, nv).
|
||||
edge(id, or).
|
||||
edge(id, ut).
|
||||
edge(id, wa).
|
||||
edge(id, wy).
|
||||
edge(il, in).
|
||||
edge(il, ky).
|
||||
edge(il, mo).
|
||||
edge(il, wi).
|
||||
edge(in, ky).
|
||||
edge(in, mi).
|
||||
edge(in, oh).
|
||||
edge(ks, mo).
|
||||
edge(ks, ne).
|
||||
edge(ks, ok).
|
||||
edge(ky, mo).
|
||||
edge(ky, oh).
|
||||
edge(ky, tn).
|
||||
edge(ky, va).
|
||||
edge(ky, wv).
|
||||
edge(la, ms).
|
||||
edge(la, tx).
|
||||
edge(ma, nh).
|
||||
edge(ma, ny).
|
||||
edge(ma, ri).
|
||||
edge(ma, vt).
|
||||
edge(md, pa).
|
||||
edge(md, va).
|
||||
edge(md, wv).
|
||||
edge(me, nh).
|
||||
edge(mi, oh).
|
||||
edge(mi, wi).
|
||||
edge(mn, nd).
|
||||
edge(mn, sd).
|
||||
edge(mn, wi).
|
||||
edge(mo, ne).
|
||||
edge(mo, ok).
|
||||
edge(mo, tn).
|
||||
edge(ms, tn).
|
||||
edge(mt, nd).
|
||||
edge(mt, sd).
|
||||
edge(mt, wy).
|
||||
edge(nc, sc).
|
||||
edge(nc, tn).
|
||||
edge(nc, va).
|
||||
edge(nd, sd).
|
||||
edge(ne, sd).
|
||||
edge(ne, wy).
|
||||
edge(nh, vt).
|
||||
edge(nj, ny).
|
||||
edge(nj, pa).
|
||||
edge(nm, ok).
|
||||
edge(nm, tx).
|
||||
edge(nv, or).
|
||||
edge(nv, ut).
|
||||
edge(ny, pa).
|
||||
edge(ny, vt).
|
||||
edge(oh, pa).
|
||||
edge(oh, wv).
|
||||
edge(ok, tx).
|
||||
edge(or, wa).
|
||||
edge(pa, wv).
|
||||
edge(sd, wy).
|
||||
edge(tn, va).
|
||||
edge(ut, wy).
|
||||
edge(va, wv).
|
||||
|
||||
independent_set(G, *(NBs)) :-
|
||||
findall(U-V, (edge(U, V),G@<U), Edges),
|
||||
setof(U, V^(member(U-V, Edges);member(V-U, Edges)), Nodes),
|
||||
pairs_keys_values(Pairs, Nodes, _),
|
||||
list_to_assoc(Pairs, Assoc),
|
||||
maplist(not_both(Assoc), Edges, NBs).
|
||||
|
||||
not_both(Assoc, U-V, ~BU + ~BV) :-
|
||||
get_assoc(U, Assoc, BU),
|
||||
get_assoc(V, Assoc, BV).
|
||||
|
||||
independent_set_count(G, Count) :- independent_set(G, Sat), sat_count(Sat, Count).
|
||||
2
benches/numlist.pl
Normal file
2
benches/numlist.pl
Normal file
@@ -0,0 +1,2 @@
|
||||
:- use_module(library(between)).
|
||||
run_numlist(Upper, Head) :- numlist(1, Upper, L), L = [Head|_].
|
||||
14
benches/run_criterion.rs
Normal file
14
benches/run_criterion.rs
Normal file
@@ -0,0 +1,14 @@
|
||||
use criterion::{criterion_group, criterion_main, Criterion};
|
||||
|
||||
mod setup;
|
||||
|
||||
fn bench_criterion(c: &mut Criterion) {
|
||||
setup::benches().run_all_criterion(c);
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
name = bench_group;
|
||||
config = Criterion::default().sample_size(10);
|
||||
targets = bench_criterion
|
||||
);
|
||||
criterion_main!(bench_group);
|
||||
21
benches/run_iai.rs
Normal file
21
benches/run_iai.rs
Normal file
@@ -0,0 +1,21 @@
|
||||
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
|
||||
|
||||
mod setup;
|
||||
|
||||
#[library_benchmark]
|
||||
#[bench::normal(setup::benches())]
|
||||
fn bench_edges(mut b: setup::Benches) {
|
||||
b.run_once("count_edges_short");
|
||||
}
|
||||
|
||||
#[library_benchmark]
|
||||
#[bench::normal(setup::benches())]
|
||||
fn bench_numlist(mut b: setup::Benches) {
|
||||
b.run_once("numlist_short");
|
||||
}
|
||||
|
||||
library_benchmark_group!(
|
||||
name = bench_group;
|
||||
benchmarks = bench_edges, bench_numlist
|
||||
);
|
||||
main!(library_benchmark_groups = bench_group);
|
||||
116
benches/setup.rs
Normal file
116
benches/setup.rs
Normal file
@@ -0,0 +1,116 @@
|
||||
use std::{collections::BTreeMap, fs, path::Path};
|
||||
|
||||
use criterion::{black_box, Criterion};
|
||||
|
||||
use maplit::btreemap;
|
||||
use scryer_prolog::machine::{
|
||||
parsed_results::{QueryMatch, QueryResolution, Value},
|
||||
Machine,
|
||||
};
|
||||
|
||||
pub fn benches() -> Benches {
|
||||
Benches::new(&[
|
||||
(
|
||||
"benches/edges.pl", // name of the prolog module file to load
|
||||
&[
|
||||
(
|
||||
"count_edges_short", // name of the benchmark
|
||||
"independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module
|
||||
btreemap! { "Count".to_string() => Value::try_from("2869176".to_string()).unwrap() }, // List of expected bindings
|
||||
),
|
||||
(
|
||||
"count_edges", // multiple benchmark queries can be defined per module
|
||||
"independent_set_count(aa, Count).", // consider making the query adjustable to tune the runtime
|
||||
btreemap! { "Count".to_string() => Value::try_from("211954906".to_string()).unwrap(), },
|
||||
),
|
||||
],
|
||||
),
|
||||
(
|
||||
"benches/numlist.pl",
|
||||
&[(
|
||||
"numlist_short",
|
||||
"run_numlist(1000000, Head).",
|
||||
btreemap! { "Head".to_string() => Value::try_from("1".to_string()).unwrap()},
|
||||
)],
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
pub struct Benches {
|
||||
machines: Vec<Machine>,
|
||||
runs: BTreeMap<String, Run>,
|
||||
}
|
||||
|
||||
pub struct Run {
|
||||
machine_idx: usize,
|
||||
name: &'static str,
|
||||
query: &'static str,
|
||||
bindings: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
// Required for using a mutex. It doesn't actually send anything across threads,
|
||||
// and this is just a benchmark, so it Should Be Fine(tm). ¯\_(ツ)_/¯
|
||||
unsafe impl Send for Benches {}
|
||||
|
||||
impl Benches {
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn new(
|
||||
benches: &[(
|
||||
&'static str,
|
||||
&[(&'static str, &'static str, BTreeMap<String, Value>)],
|
||||
)],
|
||||
) -> Self {
|
||||
let mut machines = vec![];
|
||||
let mut runs = BTreeMap::new();
|
||||
|
||||
for b in benches {
|
||||
let content = fs::read_to_string(b.0).unwrap();
|
||||
let name = Path::new(b.0).file_stem().unwrap().to_str().unwrap();
|
||||
let mut machine = Machine::new_lib();
|
||||
machine.load_module_string(name, content);
|
||||
machines.push(machine);
|
||||
let idx = machines.len() - 1;
|
||||
runs.extend(b.1.iter().cloned().map(|r| {
|
||||
(
|
||||
r.0.to_string(),
|
||||
Run {
|
||||
machine_idx: idx,
|
||||
name: r.0,
|
||||
query: r.1,
|
||||
bindings: r.2,
|
||||
},
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
Benches { machines, runs }
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn run_all_criterion(&mut self, c: &mut Criterion) {
|
||||
for (_, runner) in self.runs.iter() {
|
||||
let machine = &mut self.machines[runner.machine_idx];
|
||||
c.bench_function(runner.name, |b| {
|
||||
b.iter(|| {
|
||||
Self::run(machine, runner);
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn run_once(&mut self, name: &str) {
|
||||
let runner = &self.runs[name];
|
||||
let machine = &mut self.machines[runner.machine_idx];
|
||||
Self::run(machine, runner);
|
||||
}
|
||||
|
||||
fn run(machine: &mut Machine, runner: &Run) {
|
||||
assert_eq!(
|
||||
black_box(machine.run_query(black_box(runner.query.to_string()))),
|
||||
Ok(QueryResolution::Matches(vec![QueryMatch::from(
|
||||
runner.bindings.clone()
|
||||
)]))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -60,7 +60,7 @@ impl Machine {
|
||||
}
|
||||
|
||||
pub fn run_query(&mut self, query: String) -> QueryResult {
|
||||
println!("Query: {}", query);
|
||||
// println!("Query: {}", query);
|
||||
// Parse the query so we can analyze and then call the term
|
||||
let mut parser = Parser::new(
|
||||
Stream::from_owned_string(query, &mut self.machine_st.arena),
|
||||
@@ -192,7 +192,7 @@ impl Machine {
|
||||
let outputter = printer.print();
|
||||
|
||||
let output: String = outputter.result();
|
||||
println!("Result: {} = {}", var_key.to_string(), output);
|
||||
// println!("Result: {} = {}", var_key.to_string(), output);
|
||||
|
||||
bindings.insert(var_key.to_string(), Value::try_from(output).expect("asdfs"));
|
||||
}
|
||||
@@ -445,10 +445,7 @@ mod tests {
|
||||
}
|
||||
|
||||
// Check if the block is a query
|
||||
if block.starts_with("query") {
|
||||
// Extract the query from the block
|
||||
let query = &block[5..];
|
||||
|
||||
if let Some(query) = block.strip_prefix("query") {
|
||||
i += 1;
|
||||
println!("query #{}: {}", i, query);
|
||||
// Parse and execute the query
|
||||
@@ -458,10 +455,7 @@ mod tests {
|
||||
|
||||
// Print the result
|
||||
println!("{:?}", result);
|
||||
} else if block.starts_with("consult") {
|
||||
// Extract the code from the block
|
||||
let code = &block[7..];
|
||||
|
||||
} else if let Some(code) = block.strip_prefix("consult") {
|
||||
println!("load code: {}", code);
|
||||
|
||||
// Load the code into the machine
|
||||
|
||||
Reference in New Issue
Block a user