Merge pull request #2169 from infogulch/flame

Benchmark improvements
2023-11-24 18:57:13 +01:00
parent 1fbc7f9842 4181c9fab2
commit f1458b772e
11 changed files with 367 additions and 576 deletions
--- a/.github/actions/setup-rust/cleanup.sh
+++ b/.github/actions/setup-rust/cleanup.sh
@@ -3,8 +3,11 @@ set -e

 echo Cleanup workspace build artifacts and extra target output

-# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects
-cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`
+# clean just the direct members of the current workspace, use cargo metadata to generalize to all rust projects
+cargo clean -p `cargo metadata --no-deps --offline --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`

 # remove directories in /target/ that are not named `debug` or `release`
+before=`du -s target | awk '{print $1}'`
 find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \;
+after=`du -s target | awk '{print $1}'`
+echo Deleted $(($before - $after)) bytes from target directory
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -142,7 +142,8 @@ jobs:
          cache-context: report
      - run: |
          cargo install cargo2junit --force
-          cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'`
+          # cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'`
+          cargo install iai-callgrind-runner --force --git https://github.com/iai-callgrind/iai-callgrind --rev c77bc3c83d7f4e976cc42d4597236a8db259e772
          sudo apt install valgrind -y

      - name: Test and report
@@ -161,21 +162,20 @@ jobs:
          fail_on: nothing
          comment_mode: off

-      - run: cargo build --bench run_iai --release
-      - run: cargo bench --bench run_iai
+      - run: cargo build --all-targets --release
+      - run: cargo test --bench setup --release
+      - run: cargo bench --bench run_iai -- --save-summary=json
      - run: cargo bench --bench run_criterion
+      - run: cargo bench --bench run_criterion -- --profile-time 60

-      - name: Publish criterion benchmark results
+      - name: Publish benchmark results
        uses: actions/upload-artifact@v3
        with:
-          name: bench-criterion-results
-          path: target/criterion/**/*
-
-      - name: Publish iai benchmark results
-        uses: actions/upload-artifact@v3
-        with:
-          name: bench-iai-results
-          path: target/iai/scryer-prolog/run_iai/bench_group/*
+          name: benchmark-results
+          path: |
+            target/criterion/*
+            target/iai/*
+            target/benchmark_inference_counts.json

  # Publish binaries when building for a tag
  release:
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,6 +17,19 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"

+[[package]]
+name = "ahash"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
+dependencies = [
+ "cfg-if",
+ "getrandom",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "1.0.2"
@@ -59,6 +72,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"

+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
 [[package]]
 name = "assert_cmd"
 version = "1.0.8"
@@ -234,6 +253,12 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"

+[[package]]
+name = "bytemuck"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -370,6 +395,15 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"

+[[package]]
+name = "cpp_demangle"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "cpu-time"
 version = "1.0.0"
@@ -610,6 +644,15 @@ dependencies = [
 "num-order",
 ]

+[[package]]
+name = "debugid"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
+dependencies = [
+ "uuid",
+]
+
 [[package]]
 name = "derive_deref"
 version = "1.1.1"
@@ -701,6 +744,12 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"

+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "errno"
 version = "0.3.3"
@@ -749,6 +798,18 @@ dependencies = [
 "windows-sys",
 ]

+[[package]]
+name = "findshlibs"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64"
+dependencies = [
+ "cc",
+ "lazy_static",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -965,7 +1026,7 @@ dependencies = [
 "futures-sink",
 "futures-util",
 "http",
- "indexmap",
+ "indexmap 1.9.3",
 "slab",
 "tokio",
 "tokio-util",
@@ -1136,9 +1197,8 @@ dependencies = [

 [[package]]
 name = "iai-callgrind"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2679de583a9f5232b45b1f3a31b5e2f739bccfee9b7902488aca8f8c2c5482d1"
+version = "0.8.0"
+source = "git+https://github.com/iai-callgrind/iai-callgrind.git?rev=c77bc3c83d7f4e976cc42d4597236a8db259e772#c77bc3c83d7f4e976cc42d4597236a8db259e772"
 dependencies = [
 "bincode",
 "iai-callgrind-macros",
@@ -1148,8 +1208,7 @@ dependencies = [
 [[package]]
 name = "iai-callgrind-macros"
 version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af5af66b85e350097b8c0f6329c6347d3323d010443475741c29a1a167f116fb"
+source = "git+https://github.com/iai-callgrind/iai-callgrind.git?rev=c77bc3c83d7f4e976cc42d4597236a8db259e772#c77bc3c83d7f4e976cc42d4597236a8db259e772"
 dependencies = [
 "proc-macro-error",
 "proc-macro2",
@@ -1159,9 +1218,8 @@ dependencies = [

 [[package]]
 name = "iai-callgrind-runner"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12cc1093f263249d7c541c53849dac74f4ae756cd31e3cac1a12204efcec68bd"
+version = "0.8.0"
+source = "git+https://github.com/iai-callgrind/iai-callgrind.git?rev=c77bc3c83d7f4e976cc42d4597236a8db259e772#c77bc3c83d7f4e976cc42d4597236a8db259e772"
 dependencies = [
 "serde",
 ]
@@ -1209,6 +1267,34 @@ dependencies = [
 "hashbrown 0.12.3",
 ]

+[[package]]
+name = "indexmap"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+]
+
+[[package]]
+name = "inferno"
+version = "0.11.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c50453ec3a6555fad17b1cd1a80d16af5bc7cb35094f64e429fd46549018c6a3"
+dependencies = [
+ "ahash",
+ "indexmap 2.0.1",
+ "is-terminal",
+ "itoa",
+ "log",
+ "num-format",
+ "once_cell",
+ "quick-xml",
+ "rgb",
+ "str_stack",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -1290,7 +1376,7 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
 dependencies = [
- "arrayvec",
+ "arrayvec 0.5.2",
 "bitflags 1.3.2",
 "cfg-if",
 "ryu",
@@ -1404,6 +1490,15 @@ version = "2.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"

+[[package]]
+name = "memmap2"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a5a03cefb0d953ec0be133036f14e109412fa594edc2f77227249db66cc3ed"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.9.0"
@@ -1572,6 +1667,16 @@ dependencies = [
 "winapi",
 ]

+[[package]]
+name = "num-format"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3"
+dependencies = [
+ "arrayvec 0.7.4",
+ "itoa",
+]
+
 [[package]]
 name = "num-modular"
 version = "0.6.1"
@@ -1888,6 +1993,28 @@ dependencies = [
 "plotters-backend",
 ]

+[[package]]
+name = "pprof"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb"
+dependencies = [
+ "backtrace",
+ "cfg-if",
+ "criterion",
+ "findshlibs",
+ "inferno",
+ "libc",
+ "log",
+ "nix 0.26.4",
+ "once_cell",
+ "parking_lot 0.12.1",
+ "smallvec",
+ "symbolic-demangle",
+ "tempfile",
+ "thiserror",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -1966,6 +2093,15 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "quick-xml"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.33"
@@ -2148,6 +2284,15 @@ dependencies = [
 "winreg",
 ]

+[[package]]
+name = "rgb"
+version = "0.8.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8"
+dependencies = [
+ "bytemuck",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -2331,7 +2476,7 @@ dependencies = [
 "git-version",
 "hostname",
 "iai-callgrind",
- "indexmap",
+ "indexmap 1.9.3",
 "js-sys",
 "lazy_static",
 "lexical",
@@ -2344,6 +2489,7 @@ dependencies = [
 "num-order",
 "ordered-float",
 "phf 0.9.0",
+ "pprof",
 "predicates-core",
 "proc-macro2",
 "quote",
@@ -2627,6 +2773,12 @@ version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"

+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -2639,6 +2791,12 @@ version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0"

+[[package]]
+name = "str_stack"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
+
 [[package]]
 name = "string_cache"
 version = "0.8.7"
@@ -2696,6 +2854,29 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"

+[[package]]
+name = "symbolic-common"
+version = "12.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "405af7bd5edd866cef462e22ef73f11cf9bf506c9d62824fef8364eb69d4d4ad"
+dependencies = [
+ "debugid",
+ "memmap2",
+ "stable_deref_trait",
+ "uuid",
+]
+
+[[package]]
+name = "symbolic-demangle"
+version = "12.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bcd041ccfb77d9c70639efcd5b804b508ac7a273e9224d227379e225625daf9"
+dependencies = [
+ "cpp_demangle",
+ "rustc-demangle",
+ "symbolic-common",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -3037,6 +3218,12 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"

+[[package]]
+name = "uuid"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -3342,3 +3529,23 @@ name = "xmlparser"
 version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd"
+
+[[package]]
+name = "zerocopy"
+version = "0.7.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -112,12 +112,23 @@ assert_cmd = "1.0.3"
 predicates-core = "1.0.2"
 maplit = "1.0.2"
 serial_test = "2.0.0"
-iai-callgrind = "0.7.3"
+iai-callgrind = { git = "https://github.com/iai-callgrind/iai-callgrind.git", rev = "c77bc3c83d7f4e976cc42d4597236a8db259e772" }
 criterion = "0.5.1"

+[target.'cfg(not(target_os = "windows"))'.dev-dependencies]
+pprof = { version = "0.13.0", features = ["criterion", "flamegraph"] }
+
 [patch.crates-io]
 modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }

+[profile.bench]
+lto = true
+opt-level = 3
+
+[profile.release]
+lto = true
+opt-level = 3
+
 [[bench]]
 name = "run_criterion"
 harness = false
--- a/benches/README.md
+++ b/benches/README.md
@@ -4,16 +4,16 @@ The `benches` directory contains benchmarks that test scryer-prolog performance.

 Benchmarks are run via two harnesses:

+* `criterion` - criterion performs statistical analysis of benchmark runs and is
+  great for benchmarking locally.
 * `iai-callgrind` - this runs the benchmark with callgrind, which is able to
  precisely track the number of instructions executed during the run. This is
  especially helpful in a public CI runner context where neighboring VMs can
-  cause a very high wall time variance. This means that it doesn't track wall
-  time which is what we really care about, but it is a good tradeoff for CI
-  where tracking runtime is unreliable.
-* `criterion` - criterion performs statistical analysis of benchmark runs and is
-  great for benchmarking locally.
+  cause a very high wall time variance. While instructions executed is only
+  correlated with the desired metric (wall time), this is a good tradeoff for CI
+  where that metric is unreliable.

-Run them using the following commands:
+Run benchmarks with the following commands:

 ```
 cargo bench --bench run_criterion
@@ -21,6 +21,9 @@ cargo bench --bench run_criterion
 # run a particular criterion benchmark
 cargo bench --bench run_criterion -- <benchmark_name>

+# run in profiling mode which outputs flamegraphs. Set profile time in seconds:
+cargo bench --bench run_criterion -- --profile-time <time>
+
 # to run iai, you need valgrind installed and to install iai-callgrind-runner
 # at the same version as is in Cargo.toml:
 cargo install iai-callgrind-runner --version 0.7.3
@@ -29,7 +32,7 @@ cargo bench --bench run_iai
 ```

 For consistency, both runners -- `run_iai.rs` and `run_criterion.rs` -- import
-the same setup code from `benches.rs`.
+the same setup code from `setup.rs`.

 ## Setup

@@ -50,45 +53,43 @@ This design is meant to suppoort defining lots of benchmarks.

 To add a new benchmark:

-* Add a new file `benches/[module].pl` that contains setup code. Import
+* Add a new file `benches/[module].pl` that contains setup prolog code. Import
  libraries, define predicates, etc.
-* Add a new section in `setup.rs::benches()` that refers to it and add some
-  benchmarks.
+* Add a new section in `setup.rs::prolog_benchmarks()` that refers to to the
+  file and write a query to be benchmarked.
+* If the query mutates the machine, then use `Strategy::Fresh` so the criterion
+  benchmark will recreate a new machine for each benchmark run, otherwise use
+  `Strategy::Reuse` which has lower overhead. (This is not used by the iai
+  benchmark because it only runs once anyway.)

 Some tips:

 * The goal of benchmarking is to know if a library or engine change improved
  performance or not.
-* Once a benchmark is defined and named, avoid changing it's definition. In
-  general, if a benchmark needs to change to be more useful, give the new
-  definition a new name. This will prevent charts from showing wild changes in
+* Once a benchmark is defined and named, avoid changing it's definition. If a
+  benchmark needs to change to be more useful, give the new definition a new
+  name instead. This will prevent charts from showing wild changes in
  performance just because the definition changed (see previous).
 * Aim for queries to execute in less than 0.5s realtime. Longer runtimes make it
  easier for humans to see big differences, but benchmarks either run 10x slower
  (iai) or execute repeatedly to attain statistical significance (criterion) and
-  in both cases queries that take longer become cumbersome to run.
+  in both cases benchmarking queries that take longer than about 0.5s are
+  cumbersome to run.
 * Consider that the library runtime actually parses the text output of the top
  level. So don't use custom outputs or it will fail to parse. Also keep the
  output small so it doesn't just benchmark the ouput parsing code.
 * DO test the output of the benchmark run, we don't want to count broken
  benchmarks.
-* Because a query may run against the same machine multiple times, don't
-  permanently mutate the state of the engine with the query since that will
-  taint subsequent runs. (Benchmarking assertz et al is desirable, but will
-  require some adjustments to how the machine is set up for runs.)

 ## CI

 Both benchmark harnesses are run in `.github/workflows/ci.yaml` in the `report`
 job, and the results are published as build artifacts.

-A future action may consume the build artifacts and publish a report using the
-results.
-
 ## Todo

 - [ ] Currently, the execution time to load a module is not benchmarked. It
  would be nice to have at least one benchmark for loading a module (probably a
  big one).
- [ ] Write a new action that consumes the test and benchmark results and plots
-  them over time and publishes a report (github pages?).
+- [ ] Write a new action that downloads the test and benchmark results
+  artifacts, plots them over time, and publishes a report to github pages.
--- a/benches/run_criterion.rs
+++ b/benches/run_criterion.rs
@@ -1,5 +1,8 @@
 use criterion::{criterion_group, criterion_main, BatchSize, Criterion};

+#[cfg(not(target_os = "windows"))]
+use pprof::criterion::{Output, PProfProfiler};
+
 mod setup;

 fn bench_criterion(c: &mut Criterion) {
@@ -13,9 +16,21 @@ fn bench_criterion(c: &mut Criterion) {
    }
 }

+#[cfg(not(target_os = "windows"))]
+fn config() -> Criterion {
+    Criterion::default()
+        .sample_size(20)
+        .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)))
+}
+
+#[cfg(target_os = "windows")]
+fn config() -> Criterion {
+    Criterion::default().sample_size(20)
+}
+
 criterion_group!(
-    name = bench_group;
-    config = Criterion::default().sample_size(10);
+    name = benches;
+    config = config();
    targets = bench_criterion
 );
-criterion_main!(bench_group);
+criterion_main!(benches);
--- a/benches/run_iai.rs
+++ b/benches/run_iai.rs
@@ -1,21 +1,18 @@
 use iai_callgrind::{library_benchmark, library_benchmark_group, main};
+use scryer_prolog::machine::parsed_results::QueryResolution;

 mod setup;

 #[library_benchmark]
-#[bench::normal(setup::prolog_benches()["count_edges_short"].setup())]
-fn bench_edges(mut run: impl FnMut()) {
-    run();
-}
-
-#[library_benchmark]
-#[bench::normal(setup::prolog_benches()["numlist_short"].setup())]
-fn bench_numlist(mut run: impl FnMut()) {
-    run();
+#[bench::count_edges(setup::prolog_benches()["count_edges"].setup())]
+#[bench::numlist(setup::prolog_benches()["numlist"].setup())]
+#[bench::csv_codename(setup::prolog_benches()["csv_codename"].setup())]
+fn bench(mut run: impl FnMut() -> QueryResolution) -> QueryResolution {
+    run()
 }

 library_benchmark_group!(
-    name = bench_group;
-    benchmarks = bench_edges, bench_numlist
+    name = benches;
+    benchmarks = bench
 );
-main!(library_benchmark_groups = bench_group);
+main!(library_benchmark_groups = benches);
--- a/benches/setup.rs
+++ b/benches/setup.rs
@@ -2,7 +2,7 @@ use std::{collections::BTreeMap, fs, path::Path};

 use maplit::btreemap;
 use scryer_prolog::machine::{
-    parsed_results::{QueryMatch, QueryResolution, Value},
+    parsed_results::{QueryResolution, Value},
    Machine,
 };

@@ -10,20 +10,13 @@ pub fn prolog_benches() -> BTreeMap<&'static str, PrologBenchmark> {
    [
        (
            "count_edges",                       // name of the benchmark
-            "benches/edges.pl",                  // name of the prolog module file to load
-            "independent_set_count(aa, Count).", // query to benchmark in the context of the loaded module
-            Strategy::Reuse,
-            btreemap! { "Count" => Value::try_from("211954906".to_string()).unwrap(), }, // list of expected bindings
-        ),
-        (
-            "count_edges_short",
-            "benches/edges.pl", // use the same file in multiple benchmarks
-            "independent_set_count(ky, Count).", // consider making the query adjustable to tune the run time to ~0.1s
+            "benches/edges.pl", // name of the prolog module file to load. use the same file in multiple benchmarks
+            "independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module. consider making the query adjustable to tune the run time to ~0.1s
            Strategy::Reuse,
            btreemap! { "Count" => Value::try_from("2869176".to_string()).unwrap() },
        ),
        (
-            "numlist_short",
+            "numlist",
            "benches/numlist.pl",
            "run_numlist(1000000, Head).",
            Strategy::Reuse,
@@ -67,27 +60,73 @@ pub struct PrologBenchmark {
 }

 impl PrologBenchmark {
-    pub fn setup(&self) -> impl FnMut() {
+    pub fn make_machine(&self) -> Machine {
        let program = fs::read_to_string(self.filename).unwrap();
        let module_name = Path::new(self.filename)
            .file_stem()
            .and_then(|s| s.to_str())
            .unwrap();
-
        let mut machine = Machine::new_lib();
        machine.load_module_string(module_name, program);
+        machine
+    }

-        let benchmark_name = self.name;
+    pub fn setup(&self) -> impl FnMut() -> QueryResolution {
+        let mut machine = self.make_machine();
        let query = self.query;
-        let expected = QueryResolution::Matches(vec![QueryMatch::from(self.bindings.clone())]);
-
        move || {
            use criterion::black_box;
-            let result = black_box(machine.run_query(black_box(query.to_string())));
-            match result {
-                Ok(r) => assert_eq!(&r, &expected),
-                Err(e) => panic!("benchmark {} failed with: {}", benchmark_name, e),
-            }
+            black_box(machine.run_query(black_box(query.to_string()))).unwrap()
        }
    }
 }
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn validate_benchmarks() {
+        use super::prolog_benches;
+        use scryer_prolog::machine::parsed_results::{QueryMatch, QueryResolution};
+        use std::{fmt::Write, fs};
+
+        struct BenchResult {
+            pub name: &'static str,
+            pub setup_inference_count: u64,
+            pub query_inference_count: u64,
+        }
+
+        let mut results: Vec<BenchResult> = vec![];
+
+        for (_, r) in prolog_benches() {
+            let mut machine = r.make_machine();
+            let setup_inference_count = machine.get_inference_count();
+
+            let result = machine.run_query(r.query.to_string()).unwrap();
+            let query_inference_count = machine.get_inference_count() - setup_inference_count;
+
+            let expected = QueryResolution::Matches(vec![QueryMatch::from(r.bindings.clone())]);
+            assert_eq!(result, expected, "validating benchmark {}", r.name);
+
+            results.push(BenchResult {
+                name: r.name,
+                setup_inference_count,
+                query_inference_count,
+            })
+        }
+
+        let mut json: String = Default::default();
+        json.push('[');
+        for r in results {
+            json.push('\n');
+            write!(
+                json,
+                r#"{{"name":"{}","setup_inference_count":{},"query_inference_count":{}}},"#,
+                r.name, r.setup_inference_count, r.query_inference_count
+            )
+            .unwrap();
+        }
+        json.pop(); // trailing comma
+        json.push_str("\n]");
+        fs::write("target/benchmark_inference_counts.json", json).expect("Unable to write file");
+    }
+}
--- a/flamegraph.svg
+++ b/flamegraph.svg
--- a/log.txt
+++ b/log.txt
--- a/src/machine/mod.rs
+++ b/src/machine/mod.rs
@@ -211,6 +211,15 @@ impl Machine {
        )
    }

+    pub fn get_inference_count(&mut self) -> u64 {
+        self.machine_st
+            .cwil
+            .global_count
+            .clone()
+            .try_into()
+            .unwrap()
+    }
+
    pub fn throw_session_error(&mut self, err: SessionError, key: PredicateKey) {
        let err = self.machine_st.session_error(err);
        let stub = functor_stub(key.0, key.1);