(fix) rocksdb-compression-on-windows (sofastack#22)

* (fix) Seems like the rocksdb jni for Windows doesn't come linked with any of the compression type

* (feat) Export RocksDB options to users sofastack#20

* (fix) use Utils#cpus() to get available processors

* (fix) rm RocksConfigs

* (fix) typo

* (feat) speeding up the build with TESTFOLDER

* (feat) simplify the rheakv unit test

* (fix) rm shutdown hook with rocksdb options

.travis.yml

@@ -2,8 +2,13 @@ language: java
 sudo: false
 
 jdk:
-- openjdk11
-- openjdk8
+  - openjdk8
+  - openjdk11
+
+env:
+  - TESTFOLDER=jraft-core
+  - TESTFOLDER=jraft-rheakv/rheakv-core
+  - TESTFOLDER=jraft-rheakv/rheakv-pd
 
 cache:
   directories:
@@ -16,4 +21,4 @@ script:
   - sh ./.middleware-common/check_format.sh
 
 after_success:
-  - travis_retry mvn clean test
+  - travis_retry mvn --projects $TESTFOLDER clean test

jraft-core/src/main/java/com/alipay/sofa/jraft/core/NodeImpl.java

@@ -2506,7 +2506,7 @@ public Status transferLeadershipTo(PeerId peer) {
                 // elected. If no add_peer with this very |peer| is to be invoked ever
                 // after nor this peer is to be killed, this peer will spin in the voting
                 // procedure and make the each new leader stepped down when the peer
-                // reached vote timedout and it starts to vote (because it will increase
+                // reached vote timeout and it starts to vote (because it will increase
                 // the term of the group)
                 // To make things simple, refuse the operation and force users to
                 // invoke transfer_leadership_to after configuration changing is
@@ -2528,7 +2528,7 @@ public Status transferLeadershipTo(PeerId peer) {
                 }
             }
             if (peerId.equals(this.serverId)) {
-                LOG.info("Node {} transfered leadership to self.");
+                LOG.info("Node {} transferred leadership to self.");
                 return Status.OK();
             }
             if (!conf.contains(peerId)) {

jraft-core/src/main/java/com/alipay/sofa/jraft/core/Replicator.java

@@ -1543,7 +1543,7 @@ private boolean transferLeadership(long logIndex) {
             this.sendTimeoutNow(true, false);
             return true;
         }
-        // Register log_index so that _on_rpc_returne trigger
+        // Register log_index so that _on_rpc_return trigger
         // _send_timeout_now if _next_index reaches log_index
         this.timeoutNowIndex = logIndex;
         id.unlock();

jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/RocksDBLogStorage.java

@@ -29,8 +29,6 @@
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.ColumnFamilyOptions;
-import org.rocksdb.CompactionStyle;
-import org.rocksdb.CompressionType;
 import org.rocksdb.DBOptions;
 import org.rocksdb.IndexType;
 import org.rocksdb.Options;
@@ -54,6 +52,7 @@
 import com.alipay.sofa.jraft.option.RaftOptions;
 import com.alipay.sofa.jraft.storage.LogStorage;
 import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.StorageOptionsFactory;
 import com.alipay.sofa.jraft.util.Utils;
 
 /**
@@ -88,16 +87,15 @@ private interface WriteBatchTemplate {
     private RocksDB                         db;
     private DBOptions                       dbOptions;
     private WriteOptions                    writeOptions;
-    private final List<ColumnFamilyOptions> cfOptions         = new ArrayList<>();
+    private final List<ColumnFamilyOptions> cfOptions     = new ArrayList<>();
     private ColumnFamilyHandle              defaultHandle;
     private ColumnFamilyHandle              confHandle;
     private ReadOptions                     totalOrderReadOptions;
-    private static final int                MAX_LOG_FILE_SIZE = 1024 * 1024 * 1024;
-    private final ReadWriteLock             lock              = new ReentrantReadWriteLock(false);
-    private final Lock                      readLock          = lock.readLock();
-    private final Lock                      writeLock         = lock.writeLock();
+    private final ReadWriteLock             lock          = new ReentrantReadWriteLock(false);
+    private final Lock                      readLock      = lock.readLock();
+    private final Lock                      writeLock     = lock.writeLock();
 
-    private volatile long                   firstLogIndex     = 1;
+    private volatile long                   firstLogIndex = 1;
 
     private volatile boolean                hasLoadFirstLogIndex;
 
@@ -118,37 +116,16 @@ private static BlockBasedTableConfig createTableConfig() {
     }
 
     public static DBOptions createDBOptions() {
-        // Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
-        // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java
-        final DBOptions options = new DBOptions();
-        return options.setCreateIfMissing(true). //
-            setCreateMissingColumnFamilies(true). //
-            setMaxOpenFiles(-1). //
-            setMaxLogFileSize(MAX_LOG_FILE_SIZE). //
-            setMaxBackgroundFlushes(1). //
-            setMaxBackgroundCompactions(1);
-
+        return StorageOptionsFactory.getRocksDBOptions(RocksDBLogStorage.class);
     }
 
     public static ColumnFamilyOptions createColumnFamilyOptions() {
-        final BlockBasedTableConfig tconfig = createTableConfig();
-        final ColumnFamilyOptions options = new ColumnFamilyOptions();
-        return options.setMaxWriteBufferNumber(2). //
-            useFixedLengthPrefixExtractor(8). //
-            setTableFormatConfig(tconfig). //
-            setCompressionType(CompressionType.LZ4_COMPRESSION). //
-            setCompactionStyle(CompactionStyle.LEVEL). //
-            optimizeLevelStyleCompaction(). //
-            setLevel0FileNumCompactionTrigger(10). //
-            setLevel0SlowdownWritesTrigger(20). //
-            setLevel0StopWritesTrigger(40). //
-            setWriteBufferSize(64 * SizeUnit.MB). //
-            setMaxWriteBufferNumber(3). //
-            setTargetFileSizeBase(64 * SizeUnit.MB). //
-            setMaxBytesForLevelBase(512 * SizeUnit.MB). //
-            setMergeOperator(new StringAppendOperator()). //
-            setMemtablePrefixBloomSizeRatio(0.125);
-
+        final BlockBasedTableConfig tConfig = createTableConfig();
+        final ColumnFamilyOptions options = StorageOptionsFactory
+            .getRocksDBColumnFamilyOptions(RocksDBLogStorage.class);
+        return options.useFixedLengthPrefixExtractor(8). //
+            setTableFormatConfig(tConfig). //
+            setMergeOperator(new StringAppendOperator());
     }
 
     @Override
@@ -183,7 +160,7 @@ private boolean initAndLoad(ConfigurationManager confManager) throws RocksDBExce
         final ColumnFamilyOptions cfOption = createColumnFamilyOptions();
         this.cfOptions.add(cfOption);
         columnFamilyDescriptors.add(new ColumnFamilyDescriptor("Configuration".getBytes(), cfOption));
-        //default column family
+        // default column family
         columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOption));
 
         openDB(columnFamilyDescriptors);