bench: add bench action

.github/workflows/bench.yml

@@ -1,10 +1,15 @@
+# Do not run this workflow on pull request since this workflow has permission to modify contents.
 on:
-  workflow_dispatch:
-    inputs:
-      reason:
-        description: 'reason to trigger this build'
-        required: false
-
+  push:
+    branches:
+      - master
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
 name: Bench
 
 jobs:
@@ -26,55 +31,14 @@ jobs:
         with:
           toolchain: nightly
           default: true
-      - uses: actions-rs/cargo@v1
-        name: Benchmark 🚀
-        with:
-          command: bench
-          args: --all-features --workspace  
-  sanitizer_bench:
-    name: Bench with Sanitizer
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        name: Checkout 🛎️
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-sanitizer-bench
-      - uses: actions-rs/toolchain@v1
-        name: Setup Cargo Toolchain 🛎️
-        with:
-          components: rust-src
-          toolchain: nightly
-          default: true
-      - uses: actions-rs/cargo@v1
-        name: Bench with Address Sanitizer 🚀
-        with:
-          command: bench
-          args: --all-features -Zbuild-std --target x86_64-unknown-linux-gnu
-        env:
-          RUSTFLAGS: "-Zsanitizer=address"
-      - uses: actions-rs/cargo@v1
-        name: Bench with Leak Sanitizer 🚀
-        with:
-          command: bench
-          args: --all-features -Zbuild-std --target x86_64-unknown-linux-gnu
-        env:
-          RUSTFLAGS: "-Zsanitizer=leak"
-      - uses: actions-rs/cargo@v1
-        name: Bench with Memory Sanitizer 🚀
-        with:
-          command: bench
-          args: --all-features -Zbuild-std --target x86_64-unknown-linux-gnu
-        env:
-          RUSTFLAGS: "-Zsanitizer=memory"
-      - uses: actions-rs/cargo@v1
-        name: Bench with Thread Sanitizer 🚀
-        with:
-          command: bench
-          args: --all-features -Zbuild-std --target x86_64-unknown-linux-gnu
-        env:
-          RUSTFLAGS: "-Zsanitizer=thread"
+      - name: Benchmark 🚀
+        run: cargo +nightly bench --all-features --workspace --bench benches_badger_rocks -- --output-format bencher | tee output.txt
+      - uses: benchmark-action/github-action-benchmark@v1
+        name: Store benchmark result
+        with:
+          tool: "cargo"
+          output-file-path: output.txt
+          # Access token to deploy GitHub Pages branch
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Push and deploy GitHub pages branch automatically
+          auto-push: true

Cargo.toml

@@ -4,6 +4,10 @@ version = "0.1.0"
 authors = ["Jay Lee <[email protected]>"]
 edition = "2018"
 
+[features]
+default = []
+enable-rocksdb = ["rocksdb"]
+
 [dependencies]
 bytes = "1.0"
 coarsetime = "0.1.22"
@@ -20,6 +24,7 @@ parking_lot = "0.11"
 prost = "0.8"
 proto = { path = "proto" }
 rand = "0.7"
+rocksdb = { version = "0.15", optional = true }
 skiplist = { path = "skiplist" }
 tempdir = "0.3"
 thiserror = "1.0"
@@ -51,6 +56,10 @@ harness = false
 name = "bench_iterator"
 harness = false
 
+[[bench]]
+name = "benches_badger_rocks"
+harness = false
+
 [profile.bench]
 opt-level = 3
 debug = false
@@ -60,3 +69,6 @@ incremental = false
 debug-assertions = false
 overflow-checks = false
 rpath = false
+
+[lib]
+bench = false

agate_bench/src/main.rs

@@ -134,6 +134,7 @@ fn main() {
 
             let mut options = AgateOptions {
                 create_if_not_exists: true,
+                sync_writes: true,
                 dir: directory.clone(),
                 value_dir: directory,
                 managed_txns: true,

benches/benches_badger_rocks.rs

@@ -0,0 +1,255 @@
+#![cfg(feature = "enable-rocksdb")]
+mod common;
+
+use agatedb::Agate;
+use agatedb::AgateOptions;
+use agatedb::IteratorOptions;
+use common::{gen_kv_pair, remove_files, unix_time};
+use criterion::{criterion_group, criterion_main, Criterion};
+use rand::prelude::ThreadRng;
+use rand::Rng;
+use rocksdb::DB;
+use std::ops::Add;
+use std::sync::Arc;
+use std::time::Duration;
+use std::time::Instant;
+use tempdir::TempDir;
+
+const BATCH_SIZE: u64 = 1000;
+const SMALL_VALUE_SIZE: usize = 32;
+const LARGE_VALUE_SIZE: usize = 102400;
+
+fn badger_populate(agate: Arc<Agate>, value_size: usize) {
+    let mut txn = agate.new_transaction_at(unix_time(), true);
+
+    for i in 0..BATCH_SIZE {
+        let (key, value) = gen_kv_pair(i, value_size);
+        txn.set(key, value).unwrap();
+    }
+
+    txn.commit_at(unix_time()).unwrap();
+}
+
+fn badger_randread(agate: Arc<Agate>, value_size: usize, rng: &mut ThreadRng) {
+    let txn = agate.new_transaction_at(unix_time(), false);
+
+    for _ in 0..BATCH_SIZE {
+        let (key, value) = gen_kv_pair(rng.gen_range(0, BATCH_SIZE), value_size);
+
+        let item = txn.get(&key).unwrap();
+        assert_eq!(item.value(), value);
+    }
+}
+
+fn badger_iterate(agate: Arc<Agate>, value_size: usize) {
+    let txn = agate.new_transaction_at(unix_time(), false);
+    let opts = IteratorOptions::default();
+    let mut iter = txn.new_iterator(&opts);
+    iter.rewind();
+
+    while iter.valid() {
+        let item = iter.item();
+        assert_eq!(item.value().len(), value_size);
+
+        iter.next();
+    }
+}
+
+fn rocks_populate(db: Arc<DB>, value_size: usize) {
+    let mut write_options = rocksdb::WriteOptions::default();
+    write_options.set_sync(true);
+    write_options.disable_wal(false);
+
+    let mut batch = rocksdb::WriteBatch::default();
+
+    for i in 0..BATCH_SIZE {
+        let (key, value) = gen_kv_pair(i, value_size);
+        batch.put(key, value);
+    }
+
+    db.write_opt(batch, &write_options).unwrap();
+}
+
+fn rocks_randread(db: Arc<DB>, value_size: usize, rng: &mut ThreadRng) {
+    for _ in 0..BATCH_SIZE {
+        let (key, value) = gen_kv_pair(rng.gen_range(0, BATCH_SIZE), value_size);
+
+        let find = db.get(key).unwrap();
+        assert_eq!(find.unwrap(), value)
+    }
+}
+
+fn rocks_iterate(db: Arc<DB>, value_size: usize) {
+    let iter = db.iterator(rocksdb::IteratorMode::Start);
+
+    for (_, value) in iter {
+        assert_eq!(value.len(), value_size);
+    }
+}
+
+fn bench_badger(c: &mut Criterion) {
+    let mut rng = rand::thread_rng();
+
+    let dir = TempDir::new("agatedb-bench-small-value").unwrap();
+    let dir_path = dir.path();
+    let mut opts = AgateOptions {
+        create_if_not_exists: true,
+        sync_writes: true,
+        dir: dir_path.to_path_buf(),
+        value_dir: dir_path.to_path_buf(),
+        managed_txns: true,
+        ..Default::default()
+    };
+
+    c.bench_function("badger populate small value", |b| {
+        b.iter_custom(|iters| {
+            let mut total = Duration::new(0, 0);
+
+            (0..iters).into_iter().for_each(|_| {
+                remove_files(dir_path);
+                let agate = Arc::new(opts.open().unwrap());
+
+                let now = Instant::now();
+                badger_populate(agate, SMALL_VALUE_SIZE);
+                total = total.add(now.elapsed());
+            });
+
+            total
+        });
+    });
+
+    let agate = Arc::new(opts.open().unwrap());
+
+    c.bench_function("badger randread small value", |b| {
+        b.iter(|| {
+            badger_randread(agate.clone(), SMALL_VALUE_SIZE, &mut rng);
+        });
+    });
+
+    c.bench_function("badger iterate small value", |b| {
+        b.iter(|| {
+            badger_iterate(agate.clone(), SMALL_VALUE_SIZE);
+        });
+    });
+
+    dir.close().unwrap();
+    let dir = TempDir::new("agatedb-bench-large-value").unwrap();
+    let dir_path = dir.path();
+    opts.dir = dir_path.to_path_buf();
+    opts.value_dir = dir_path.to_path_buf();
+
+    c.bench_function("badger populate large value", |b| {
+        b.iter_custom(|iters| {
+            let mut total = Duration::new(0, 0);
+
+            (0..iters).into_iter().for_each(|_| {
+                remove_files(dir_path);
+                let agate = Arc::new(opts.open().unwrap());
+
+                let now = Instant::now();
+                badger_populate(agate, LARGE_VALUE_SIZE);
+                total = total.add(now.elapsed());
+            });
+
+            total
+        });
+    });
+
+    let agate = Arc::new(opts.open().unwrap());
+
+    c.bench_function("badger randread large value", |b| {
+        b.iter(|| {
+            badger_randread(agate.clone(), LARGE_VALUE_SIZE, &mut rng);
+        });
+    });
+
+    c.bench_function("badger iterate large value", |b| {
+        b.iter(|| {
+            badger_iterate(agate.clone(), LARGE_VALUE_SIZE);
+        });
+    });
+
+    dir.close().unwrap();
+}
+
+fn bench_rocks(c: &mut Criterion) {
+    let mut rng = rand::thread_rng();
+
+    let dir = TempDir::new("rocks-bench-small-value").unwrap();
+    let dir_path = dir.path();
+    let mut opts = rocksdb::Options::default();
+    opts.create_if_missing(true);
+    opts.set_compression_type(rocksdb::DBCompressionType::None);
+
+    c.bench_function("rocks populate small value", |b| {
+        b.iter_custom(|iters| {
+            let mut total = Duration::new(0, 0);
+
+            (0..iters).into_iter().for_each(|_| {
+                remove_files(dir_path);
+                let db = Arc::new(rocksdb::DB::open(&opts, &dir).unwrap());
+
+                let now = Instant::now();
+                rocks_populate(db, SMALL_VALUE_SIZE);
+                total = total.add(now.elapsed());
+            });
+
+            total
+        });
+    });
+
+    let db = Arc::new(rocksdb::DB::open(&opts, &dir).unwrap());
+
+    c.bench_function("rocks randread small value", |b| {
+        b.iter(|| {
+            rocks_randread(db.clone(), SMALL_VALUE_SIZE, &mut rng);
+        });
+    });
+
+    c.bench_function("rocks iterate small value", |b| {
+        b.iter(|| rocks_iterate(db.clone(), SMALL_VALUE_SIZE));
+    });
+
+    dir.close().unwrap();
+    let dir = TempDir::new("rocks-bench-large-value").unwrap();
+    let dir_path = dir.path();
+
+    c.bench_function("rocks populate large value", |b| {
+        b.iter_custom(|iters| {
+            let mut total = Duration::new(0, 0);
+
+            (0..iters).into_iter().for_each(|_| {
+                remove_files(dir_path);
+                let db = Arc::new(rocksdb::DB::open(&opts, &dir).unwrap());
+
+                let now = Instant::now();
+                rocks_populate(db, LARGE_VALUE_SIZE);
+                total = total.add(now.elapsed());
+            });
+
+            total
+        });
+    });
+
+    let db = Arc::new(rocksdb::DB::open(&opts, &dir).unwrap());
+
+    c.bench_function("rocks randread large value", |b| {
+        b.iter(|| {
+            rocks_randread(db.clone(), LARGE_VALUE_SIZE, &mut rng);
+        });
+    });
+
+    c.bench_function("rocks iterate large value", |b| {
+        b.iter(|| rocks_iterate(db.clone(), LARGE_VALUE_SIZE));
+    });
+
+    dir.close().unwrap();
+}
+
+criterion_group! {
+  name = benches_badger_rocks;
+  config = Criterion::default();
+  targets = bench_badger, bench_rocks
+}
+
+criterion_main!(benches_badger_rocks);