diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml new file mode 100644 index 00000000..3495b6aa --- /dev/null +++ b/.github/actions/setup-rust/action.yml @@ -0,0 +1,52 @@ +name: 'Setup Rust' +inputs: + rust-version: + required: true + type: string + targets: + required: true + type: string + components: + required: false + default: + cache-context: + required: true + type: string + +runs: + using: "composite" + steps: + - uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: ${{ inputs.rust-version }} + targets: ${{ inputs.targets }} + components: ${{ inputs.components }} + + - name: Install i686 dependencies + if: "contains(inputs.targets,'i686')" + shell: bash + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update + sudo apt-get install libssl-dev:i386 gcc-multilib clang -y + echo "CC=clang" >> $GITHUB_ENV + echo "PKG_CONFIG_SYSROOT_DIR=/" >> $GITHUB_ENV + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ inputs.cache-context }}_${{ inputs.targets }}_rustc-${{ steps.toolchain.outputs.cachekey }}_cargo-${{ hashFiles('**/Cargo.lock') }} + + # Remove build artifacts for the current crate, since it will be rebuilt every + # run anyway, but keep dependency artifacts to cache them. + # Must be placed after actions/cache so its post step runs first. + - uses: pyTooling/Actions/with-post-step@v0.4.6 + with: + main: bash ./.github/actions/setup-rust/cleanup.sh + post: bash ./.github/actions/setup-rust/cleanup.sh diff --git a/.github/actions/setup-rust/cleanup.sh b/.github/actions/setup-rust/cleanup.sh new file mode 100755 index 00000000..fd51bc92 --- /dev/null +++ b/.github/actions/setup-rust/cleanup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +echo Cleanup workspace build artifacts and extra target output + +# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects +cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'` + +# remove directories in /target/ that are not named `debug` or `release` +find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9161052..51e97199 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,42 +15,18 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@master - id: toolchain + - name: Setup Rust + uses: ./.github/actions/setup-rust with: - toolchain: nightly + rust-version: nightly targets: x86_64-unknown-linux-gnu components: clippy, rustfmt - - run: cargo install cargo2junit --force - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: style_rustc-${{ steps.toolchain.outputs.cachekey }}_cargo-${{ hashFiles('**/Cargo.lock') }} + cache-context: style - name: Check formatting run: cargo fmt --check - name: Check clippy run: cargo clippy --no-deps --all-targets - - name: Test and report - run: | - RUSTC_BOOTSTRAP=1 cargo test --all -- -Z unstable-options --format json --report-time | cargo2junit > cargo_test_results.xml - - name: Publish cargo test results artifact - uses: actions/upload-artifact@v3 - with: - name: cargo-test-results - path: cargo_test_results.xml - - name: Publish cargo test summary - uses: EnricoMi/publish-unit-test-result-action/composite@master - with: - check_name: Cargo test summary - files: cargo_test_results.xml - fail_on: nothing - comment_mode: off build-test: runs-on: ${{ matrix.os }} @@ -75,34 +51,19 @@ jobs: shell: bash steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@master - id: toolchain + - name: Setup Rust + uses: ./.github/actions/setup-rust with: - toolchain: ${{ matrix.rust-version }} + rust-version: ${{ matrix.rust-version }} targets: ${{ matrix.target }} - - name: Install i686 dependencies - if: "contains(matrix.target,'i686')" - run: | - sudo dpkg --add-architecture i386 - sudo apt-get update - sudo apt-get install libssl-dev:i386 gcc-multilib clang -y - echo "CC=clang" >> $GITHUB_ENV - echo "PKG_CONFIG_SYSROOT_DIR=/" >> $GITHUB_ENV - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ matrix.os }}_${{ matrix.target }}_rustc-${{ steps.toolchain.outputs.cachekey }}_cargo-${{ hashFiles('**/Cargo.lock') }} + cache-context: ${{ matrix.os }} # Build and test. - name: Build library run: cargo rustc --lib --target ${{ matrix.target }} ${{ matrix.args }} --verbose - name: Test - run: cargo test --target ${{ matrix.target }} ${{ matrix.args }} --all --verbose || echo "::warning ::Tests failed" + continue-on-error: ${{ contains(matrix.target,'wasm32') }} # allow wasm builds to fail tests for now + run: cargo test --target ${{ matrix.target }} ${{ matrix.args }} --all --verbose # On stable rust builds, build a binary and publish as a github actions # artifact. These binaries could be useful for testing the pipeline but @@ -166,6 +127,53 @@ jobs: fail_on: nothing comment_mode: off + report: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - name: Setup Rust + uses: ./.github/actions/setup-rust + with: + rust-version: nightly + targets: x86_64-unknown-linux-gnu + cache-context: report + - run: | + cargo install cargo2junit --force + cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'` + sudo apt install valgrind -y + + - name: Test and report + run: | + RUSTC_BOOTSTRAP=1 cargo test --all -- -Z unstable-options --format json --report-time | cargo2junit > cargo_test_results.xml + - name: Publish cargo test results artifact + uses: actions/upload-artifact@v3 + with: + name: cargo-test-results + path: cargo_test_results.xml + - name: Publish cargo test summary + uses: EnricoMi/publish-unit-test-result-action/composite@master + with: + check_name: Cargo test summary + files: cargo_test_results.xml + fail_on: nothing + comment_mode: off + + - run: cargo build --bench run_iai --release + - run: cargo bench --bench run_iai + - run: cargo bench --bench run_criterion + + - name: Publish criterion benchmark results + uses: actions/upload-artifact@v3 + with: + name: bench-criterion-results + path: target/criterion/**/* + + - name: Publish iai benchmark results + uses: actions/upload-artifact@v3 + with: + name: bench-iai-results + path: target/iai/scryer-prolog/run_iai/bench_group/* + # Publish binaries when building for a tag release: runs-on: ubuntu-20.04 diff --git a/Cargo.lock b/Cargo.lock index 315f61a7..8d8cba8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,18 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + [[package]] name = "arrayvec" version = "0.5.2" @@ -100,6 +112,15 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -225,6 +246,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.83" @@ -254,6 +281,58 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + [[package]] name = "clipboard-win" version = "4.5.0" @@ -320,6 +399,75 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossterm" version = "0.20.0" @@ -834,6 +982,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.12.3" @@ -990,6 +1144,38 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iai-callgrind" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2679de583a9f5232b45b1f3a31b5e2f739bccfee9b7902488aca8f8c2c5482d1" +dependencies = [ + "bincode", + "iai-callgrind-macros", + "iai-callgrind-runner", +] + +[[package]] +name = "iai-callgrind-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af5af66b85e350097b8c0f6329c6347d3323d010443475741c29a1a167f116fb" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "iai-callgrind-runner" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cc1093f263249d7c541c53849dac74f4ae756cd31e3cac1a12204efcec68bd" +dependencies = [ + "serde", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -1048,6 +1234,17 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1217,6 +1414,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -1425,6 +1631,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.2.3" @@ -1658,6 +1870,34 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1697,6 +1937,30 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -1767,6 +2031,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2044,6 +2328,7 @@ dependencies = [ "console_error_panic_hook", "console_log", "cpu-time", + "criterion", "crossterm", "crrl", "ctrlc", @@ -2056,6 +2341,7 @@ dependencies = [ "getrandom", "git-version", "hostname", + "iai-callgrind", "indexmap", "lazy_static", "lexical", @@ -2498,6 +2784,16 @@ dependencies = [ "syn 2.0.37", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index c5e1bab7..b36e4e13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,7 +85,13 @@ tokio = { version = "1.28.2", features = ["full"] } [target.'cfg(target_arch = "wasm32")'.dependencies] getrandom = { version = "0.2.10", features = ["js"] } -tokio = { version = "1.28.2", features = ["sync", "macros", "io-util", "rt", "time"] } +tokio = { version = "1.28.2", features = [ + "sync", + "macros", + "io-util", + "rt", + "time", +] } [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] console_error_panic_hook = "0.1" @@ -93,11 +99,7 @@ console_log = "1.0" wasm-bindgen = "0.2.87" wasm-bindgen-futures = "0.4" serde-wasm-bindgen = "0.5" -web-sys = { version = "0.3", features = [ - "Document", - "Window", - "Element", -]} +web-sys = { version = "0.3", features = ["Document", "Window", "Element"] } [target.'cfg(target_os = "wasi")'.dependencies] ring-wasi = { version = "0.16.25" } @@ -110,6 +112,16 @@ assert_cmd = "1.0.3" predicates-core = "1.0.2" maplit = "1.0.2" serial_test = "2.0.0" +iai-callgrind = "0.7.3" +criterion = "0.5.1" [patch.crates-io] -modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" } \ No newline at end of file +modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" } + +[[bench]] +name = "run_criterion" +harness = false + +[[bench]] +name = "run_iai" +harness = false diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 00000000..3716c9c0 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,95 @@ +# About benches + +The `benches` directory contains benchmarks that test scryer-prolog performance. + +Benchmarks are run via two harnesses: + +* `iai-callgrind` - this runs the benchmark with callgrind, which is able to + precisely track the number of instructions executed during the run. This is + especially helpful in a public CI runner context where neighboring VMs can + cause a very high wall time variance. This means that it doesn't track wall + time which is what we really care about, but it is a good tradeoff for CI + where tracking runtime is unreliable. +* `criterion` - criterion performs statistical analysis of benchmark runs and is + great for benchmarking locally. + +Run them using the following commands: + +``` +cargo bench --bench run_criterion + +# to run iai, you need valgrind installed and to install iai-callgrind-runner +# at the same version as is in Cargo.toml: +cargo install iai-callgrind-runner --version 0.7.3 + +cargo bench --bench run_iai +``` + +For consistency, both runners -- `run_iai.rs` and `run_criterion.rs` -- import +the same setup code from `benches.rs`. + +## Setup + +`setup.rs` contains the setup code for the actual benchmarks, which are run +using the `Benches` struct. `fn benches()` at the top of the file is where the +benchmarks are defined. + +Benchmarks are organized around running queries against one prolog module file. +Before any runs start, `Benches::new()` reads the module files and initializes a +new `scryer_prolog::machine::Machine` for each file; multiple queries can be +declared to be benchmarked in the context of that module/machine instance. + +Each benchmark measurement is done by running a query against the machine. In +the case of criterion each query is run many times, in the case of iai it's run +once. + +## Adding benchmarks + +This design is meant to suppoort defining lots of benchmarks. + +To add a new benchmark: + +* Add a new file `benches/[module].pl` that contains setup code. Import + libraries, define predicates, etc. +* Add a new section in `setup.rs::benches()` that refers to it and add some + benchmarks. + +Some tips: + +* The goal of benchmarking is to know if a library or engine change improved + performance or not. +* Once a benchmark is defined and named, don't change it's definition. If a + benchmark needs to change to be more useful, give the new definition a new + name. This will prevent charts from showing wild changes in performance just + because the definition changed (see previous). +* Aim for queries to execute in about 0.1-0.5s realtime. Longer runtimes make it + easier for humans to see big differences, but benchmarks either run 10x slower + (iai) or execute repeatedly to attain statistical significance (criterion) and + in both cases queries that take 5+ seconds quickly become unweildly. +* Consider that the library runtime actually parses the text output of the top + level. So keep the output small and don't use custom outputs or it will fail + to parse. +* DO test the output of the benchmark run, we don't want to count broken + benchmarks. +* Because a query may run against the same machine multiple times, don't + permanently mutate the state of the engine with the query since that will + taint subsequent runs. (Benchmarking assertz et al is desirable, but will + require some adjustments to how the machine is set up for runs.) + +## CI + +Both benchmark harnesses are run in `.github/workflows/ci.yaml` in the `report` +job, and the results are published as build artifacts. + +A future action may consume the build artifacts and publish a report using the +results. + +## Todo + +- [ ] Currently, the execution time to load a module is not benchmarked. It + would be nice to have at least one benchmark for loading a module (probably a + big one). +- [ ] Adjust the benchmark execution strategy to allow queries to modify the + engine state (`assertz` etc). +- [ ] Write a new action that consumes the test and benchmark results and plots + them over time and publishes a report (github pages?). diff --git a/benches/edges.pl b/benches/edges.pl new file mode 100644 index 00000000..b9529ded --- /dev/null +++ b/benches/edges.pl @@ -0,0 +1,130 @@ +:- use_module(library(clpb)). +:- use_module(library(assoc)). +:- use_module(library(lists)). +:- use_module(library(pairs)). + +/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Contiguous United States and DC as they appear in SGB: + http://www-cs-faculty.stanford.edu/~uno/sgb.html +- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */ + +edge(al, fl). +edge(al, ga). +edge(al, ms). +edge(al, tn). +edge(ar, la). +edge(ar, mo). +edge(ar, ms). +edge(ar, ok). +edge(ar, tn). +edge(ar, tx). +edge(az, ca). +edge(az, nm). +edge(az, nv). +edge(az, ut). +edge(ca, nv). +edge(ca, or). +edge(co, ks). +edge(co, ne). +edge(co, nm). +edge(co, ok). +edge(co, ut). +edge(co, wy). +edge(ct, ma). +edge(ct, ny). +edge(ct, ri). +edge(dc, md). +edge(dc, va). +edge(de, md). +edge(de, nj). +edge(de, pa). +edge(fl, ga). +edge(ga, nc). +edge(ga, sc). +edge(ga, tn). +edge(ia, il). +edge(ia, mn). +edge(ia, mo). +edge(ia, ne). +edge(ia, sd). +edge(ia, wi). +edge(id, mt). +edge(id, nv). +edge(id, or). +edge(id, ut). +edge(id, wa). +edge(id, wy). +edge(il, in). +edge(il, ky). +edge(il, mo). +edge(il, wi). +edge(in, ky). +edge(in, mi). +edge(in, oh). +edge(ks, mo). +edge(ks, ne). +edge(ks, ok). +edge(ky, mo). +edge(ky, oh). +edge(ky, tn). +edge(ky, va). +edge(ky, wv). +edge(la, ms). +edge(la, tx). +edge(ma, nh). +edge(ma, ny). +edge(ma, ri). +edge(ma, vt). +edge(md, pa). +edge(md, va). +edge(md, wv). +edge(me, nh). +edge(mi, oh). +edge(mi, wi). +edge(mn, nd). +edge(mn, sd). +edge(mn, wi). +edge(mo, ne). +edge(mo, ok). +edge(mo, tn). +edge(ms, tn). +edge(mt, nd). +edge(mt, sd). +edge(mt, wy). +edge(nc, sc). +edge(nc, tn). +edge(nc, va). +edge(nd, sd). +edge(ne, sd). +edge(ne, wy). +edge(nh, vt). +edge(nj, ny). +edge(nj, pa). +edge(nm, ok). +edge(nm, tx). +edge(nv, or). +edge(nv, ut). +edge(ny, pa). +edge(ny, vt). +edge(oh, pa). +edge(oh, wv). +edge(ok, tx). +edge(or, wa). +edge(pa, wv). +edge(sd, wy). +edge(tn, va). +edge(ut, wy). +edge(va, wv). + +independent_set(G, *(NBs)) :- + findall(U-V, (edge(U, V),G@ Benches { + Benches::new(&[ + ( + "benches/edges.pl", // name of the prolog module file to load + &[ + ( + "count_edges_short", // name of the benchmark + "independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module + btreemap! { "Count".to_string() => Value::try_from("2869176".to_string()).unwrap() }, // List of expected bindings + ), + ( + "count_edges", // multiple benchmark queries can be defined per module + "independent_set_count(aa, Count).", // consider making the query adjustable to tune the runtime + btreemap! { "Count".to_string() => Value::try_from("211954906".to_string()).unwrap(), }, + ), + ], + ), + ( + "benches/numlist.pl", + &[( + "numlist_short", + "run_numlist(1000000, Head).", + btreemap! { "Head".to_string() => Value::try_from("1".to_string()).unwrap()}, + )], + ), + ]) +} + +pub struct Benches { + machines: Vec, + runs: BTreeMap, +} + +pub struct Run { + machine_idx: usize, + name: &'static str, + query: &'static str, + bindings: BTreeMap, +} + +// Required for using a mutex. It doesn't actually send anything across threads, +// and this is just a benchmark, so it Should Be Fine(tm). ¯\_(ツ)_/¯ +unsafe impl Send for Benches {} + +impl Benches { + #[allow(clippy::type_complexity)] + pub fn new( + benches: &[( + &'static str, + &[(&'static str, &'static str, BTreeMap)], + )], + ) -> Self { + let mut machines = vec![]; + let mut runs = BTreeMap::new(); + + for b in benches { + let content = fs::read_to_string(b.0).unwrap(); + let name = Path::new(b.0).file_stem().unwrap().to_str().unwrap(); + let mut machine = Machine::new_lib(); + machine.load_module_string(name, content); + machines.push(machine); + let idx = machines.len() - 1; + runs.extend(b.1.iter().cloned().map(|r| { + ( + r.0.to_string(), + Run { + machine_idx: idx, + name: r.0, + query: r.1, + bindings: r.2, + }, + ) + })); + } + + Benches { machines, runs } + } + + #[allow(dead_code)] + pub fn run_all_criterion(&mut self, c: &mut Criterion) { + for (_, runner) in self.runs.iter() { + let machine = &mut self.machines[runner.machine_idx]; + c.bench_function(runner.name, |b| { + b.iter(|| { + Self::run(machine, runner); + }) + }); + } + } + + #[allow(dead_code)] + pub fn run_once(&mut self, name: &str) { + let runner = &self.runs[name]; + let machine = &mut self.machines[runner.machine_idx]; + Self::run(machine, runner); + } + + fn run(machine: &mut Machine, runner: &Run) { + assert_eq!( + black_box(machine.run_query(black_box(runner.query.to_string()))), + Ok(QueryResolution::Matches(vec![QueryMatch::from( + runner.bindings.clone() + )])) + ); + } +} diff --git a/src/codegen.rs b/src/codegen.rs index 21fd4e4d..7297b72c 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -296,12 +296,12 @@ impl DebrayAllocator { self.mark_var_in_non_callable(var_num, term_loc, vr, code); temp_v!(arg) } else { - match &self.var_data.records[var_num].allocation { - VarAlloc::Perm(_, PermVarAllocation::Pending) => { - self.mark_var_in_non_callable(var_num, term_loc, vr, code); - } - _ => {} - } + match &self.var_data.records[var_num].allocation { + VarAlloc::Perm(_, PermVarAllocation::Pending) => { + self.mark_var_in_non_callable(var_num, term_loc, vr, code); + } + _ => {} + } self.increment_running_count(var_num); RegType::Perm(p) diff --git a/src/debray_allocator.rs b/src/debray_allocator.rs index 30bb0a24..593e584a 100644 --- a/src/debray_allocator.rs +++ b/src/debray_allocator.rs @@ -166,12 +166,12 @@ impl DebrayAllocator { for var_num in subsumed_hits { match &mut self.var_data.records[var_num].allocation { VarAlloc::Perm(_, ref mut allocation) => { - if let PermVarAllocation::Done { + if let PermVarAllocation::Done { shallow_safety, deep_safety, .. } = allocation - { + { if !self .branch_stack .safety_unneeded_in_branch(shallow_safety, &branch_designator) @@ -188,7 +188,7 @@ impl DebrayAllocator { branch_occurrences.deep_safety.insert(var_num); } } - + *allocation = PermVarAllocation::Pending; } _ => unreachable!(), @@ -470,7 +470,7 @@ impl DebrayAllocator { #[inline(always)] pub fn get_binding(&self, var_num: usize) -> RegType { - self.var_data.records[var_num].allocation.as_reg_type() + self.var_data.records[var_num].allocation.as_reg_type() } pub fn num_perm_vars(&self) -> usize { @@ -748,7 +748,7 @@ impl Allocator for DebrayAllocator { } RegType::Perm(0) => { let p = self.alloc_perm_var(var_num, term_loc.chunk_num()); - cell.set(VarReg::Norm(RegType::Perm(p))); + cell.set(VarReg::Norm(RegType::Perm(p))); (RegType::Perm(p), true) } r @ RegType::Perm(_) => { diff --git a/src/forms.rs b/src/forms.rs index d905571f..a42dda7f 100644 --- a/src/forms.rs +++ b/src/forms.rs @@ -885,8 +885,6 @@ pub(crate) struct PredicateInfo { pub(crate) has_clauses: bool, } - - impl PredicateInfo { #[inline] pub(crate) fn compile_incrementally(&self) -> bool { diff --git a/src/machine/lib_machine.rs b/src/machine/lib_machine.rs index 3a27b3d2..43aed66a 100644 --- a/src/machine/lib_machine.rs +++ b/src/machine/lib_machine.rs @@ -60,7 +60,7 @@ impl Machine { } pub fn run_query(&mut self, query: String) -> QueryResult { - println!("Query: {}", query); + // println!("Query: {}", query); // Parse the query so we can analyze and then call the term let mut parser = Parser::new( Stream::from_owned_string(query, &mut self.machine_st.arena), @@ -192,7 +192,7 @@ impl Machine { let outputter = printer.print(); let output: String = outputter.result(); - println!("Result: {} = {}", var_key.to_string(), output); + // println!("Result: {} = {}", var_key.to_string(), output); bindings.insert(var_key.to_string(), Value::try_from(output).expect("asdfs")); } @@ -445,10 +445,7 @@ mod tests { } // Check if the block is a query - if block.starts_with("query") { - // Extract the query from the block - let query = &block[5..]; - + if let Some(query) = block.strip_prefix("query") { i += 1; println!("query #{}: {}", i, query); // Parse and execute the query @@ -458,10 +455,7 @@ mod tests { // Print the result println!("{:?}", result); - } else if block.starts_with("consult") { - // Extract the code from the block - let code = &block[7..]; - + } else if let Some(code) = block.strip_prefix("consult") { println!("load code: {}", code); // Load the code into the machine