Merge pull request ceph#12287 from liewegas/wip-store-kv-stats

kv: dump rocksdb stats

src/common/config_opts.h

@@ -886,7 +886,10 @@ OPTION(rocksdb_log_to_ceph_log, OPT_BOOL, true)  // log to ceph log
 OPTION(rocksdb_cache_size, OPT_INT, 128*1024*1024)  // default rocksdb cache size
 OPTION(rocksdb_cache_shard_bits, OPT_INT, 4)  // rocksdb block cache shard bits, 4 bit -> 16 shards
 OPTION(rocksdb_block_size, OPT_INT, 4*1024)  // default rocksdb block size
-OPTION(rocksdb_perf, OPT_BOOL, false) // rocksdb breakdown
+OPTION(rocksdb_perf, OPT_BOOL, false) // Enabling this will have 5-10% impact on performance for the stats collection
+OPTION(rocksdb_collect_compaction_stats, OPT_BOOL, false) //For rocksdb, this behavior will be an overhead of 5%~10%, collected only rocksdb_perf is enabled.
+OPTION(rocksdb_collect_extended_stats, OPT_BOOL, false) //For rocksdb, this behavior will be an overhead of 5%~10%, collected only rocksdb_perf is enabled.
+OPTION(rocksdb_collect_memory_stats, OPT_BOOL, false) //For rocksdb, this behavior will be an overhead of 5%~10%, collected only rocksdb_perf is enabled.
 
 // rocksdb options that will be used for omap(if omap_backend is rocksdb)
 OPTION(filestore_rocksdb_options, OPT_STR, "")

src/kv/KeyValueDB.h

@@ -11,6 +11,7 @@
 #include "include/memory.h"
 #include <boost/scoped_ptr.hpp>
 #include "include/encoding.h"
+#include "common/Formatter.h"
 
 using std::string;
 /**
@@ -312,6 +313,9 @@ class KeyValueDB {
     return -EOPNOTSUPP;
   }
 
+  virtual void get_statistics(Formatter *f) {
+    return;
+  }
 protected:
   /// List of matching prefixes and merge operators
   std::vector<std::pair<std::string,

src/kv/RocksDBStore.cc

@@ -22,6 +22,7 @@ using std::string;
 #include "common/perf_counters.h"
 #include "common/debug.h"
 #include "include/str_list.h"
+#include "include/stringify.h"
 #include "include/str_map.h"
 #include "KeyValueDB.h"
 #include "RocksDBStore.h"
@@ -242,6 +243,12 @@ int RocksDBStore::do_open(ostream &out, bool create_if_missing)
       return -EINVAL;
     }
   }
+
+  if (g_conf->rocksdb_perf)  {
+    dbstats = rocksdb::CreateDBStatistics();
+    opt.statistics = dbstats;
+  }
+
   opt.create_if_missing = create_if_missing;
   if (g_conf->rocksdb_separate_wal_dir) {
     opt.wal_dir = path + ".wal";
@@ -277,9 +284,8 @@ int RocksDBStore::do_open(ostream &out, bool create_if_missing)
     dout(10) << __func__ << " using custom Env " << priv << dendl;
     opt.env = static_cast<rocksdb::Env*>(priv);
   }
-  
+
   auto cache = rocksdb::NewLRUCache(g_conf->rocksdb_cache_size, g_conf->rocksdb_cache_shard_bits);
-  rocksdb::BlockBasedTableOptions bbt_opts;
   bbt_opts.block_size = g_conf->rocksdb_block_size;
   bbt_opts.block_cache = cache;
   opt.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbt_opts));
@@ -363,6 +369,47 @@ void RocksDBStore::close()
     cct->get_perfcounters_collection()->remove(logger);
 }
 
+void RocksDBStore::get_statistics(Formatter *f)
+{
+  if (!g_conf->rocksdb_perf)  {
+    dout(20) << __func__ << "RocksDB perf is disabled, can't probe for stats"
+	     << dendl;
+    return;
+  }
+
+  if (g_conf->rocksdb_collect_compaction_stats) {
+    std::string stats;
+    bool status = db->GetProperty("rocksdb.stats", &stats);
+    if (status) {
+      f->open_object_section("rocksdb_statistics");
+      f->dump_string("rocksdb_compaction_statistics", stats);
+      f->close_section();
+    }
+  }
+  if (g_conf->rocksdb_collect_extended_stats) {
+    if (dbstats) {
+      f->open_object_section("rocksdb_extended_statistics");
+      f->dump_string("rocksdb_extended_statistics", dbstats->ToString().c_str());
+      f->close_section();
+    }
+    f->open_object_section("rocksdbstore_perf_counters");
+    logger->dump_formatted(f,0);
+    f->close_section();
+  }
+  if (g_conf->rocksdb_collect_memory_stats) {
+    f->open_object_section("rocksdb_memtable_statistics");
+    std::string str(stringify(bbt_opts.block_cache->GetUsage()));
+    f->dump_string("block_cache_usage", str.data());
+    str.clear();
+    str.append(stringify(bbt_opts.block_cache->GetPinnedUsage()));
+    f->dump_string("block_cache_pinned_blocks_usage", str);
+    str.clear();
+    db->GetProperty("rocksdb.cur-size-all-mem-tables", &str);
+    f->dump_string("rocksdb_memtable_usage", str);
+    f->close_section();
+  }
+}
+
 int RocksDBStore::submit_transaction(KeyValueDB::Transaction t)
 {
   utime_t start = ceph_clock_now(g_ceph_context);

src/kv/RocksDBStore.h

@@ -13,6 +13,9 @@
 #include <boost/scoped_ptr.hpp>
 #include "rocksdb/write_batch.h"
 #include "rocksdb/perf_context.h"
+#include "rocksdb/iostats_context.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/table.h"
 #include <errno.h>
 #include "common/errno.h"
 #include "common/dout.h"
@@ -53,6 +56,7 @@ namespace rocksdb{
   class Iterator;
   class Logger;
   struct Options;
+  struct BlockBasedTableOptions;
 }
 
 extern rocksdb::Logger *create_rocksdb_ceph_logger();
@@ -67,7 +71,10 @@ class RocksDBStore : public KeyValueDB {
   void *priv;
   rocksdb::DB *db;
   rocksdb::Env *env;
+  std::shared_ptr<rocksdb::Statistics> dbstats;
+  rocksdb::BlockBasedTableOptions bbt_opts;
   string options_str;
+
   int do_open(ostream &out, bool create_if_missing);
 
   // manage async compactions
@@ -123,6 +130,7 @@ class RocksDBStore : public KeyValueDB {
     priv(p),
     db(NULL),
     env(static_cast<rocksdb::Env*>(p)),
+    dbstats(NULL),
     compact_queue_lock("RocksDBStore::compact_thread_lock"),
     compact_queue_stop(false),
     compact_thread(this),
@@ -141,6 +149,9 @@ class RocksDBStore : public KeyValueDB {
   int create_and_open(ostream &out);
 
   void close();
+
+  void get_statistics(Formatter *f);
+
   struct  RocksWBHandler: public rocksdb::WriteBatch::Handler {
     std::string seen ;
     int num_seen = 0;

src/os/ObjectStore.h

@@ -1496,6 +1496,8 @@ class ObjectStore {
     return 0;
   }
 
+  virtual void get_db_statistics(Formatter *f) { }
+
   virtual string get_type() = 0;
 
   // mgmt

src/os/bluestore/BlueStore.cc

@@ -6162,6 +6162,10 @@ uint64_t BlueStore::_assign_blobid(TransContext *txc)
   return bid;
 }
 
+void BlueStore::get_db_statistics(Formatter *f)
+{
+  db->get_statistics(f);
+}
 
 BlueStore::TransContext *BlueStore::_txc_create(OpSequencer *osr)
 {

src/os/bluestore/BlueStore.h

@@ -1753,6 +1753,8 @@ class BlueStore : public ObjectStore,
     return 0;
   }
 
+  void get_db_statistics(Formatter *f);
+
 public:
   int statfs(struct store_statfs_t *buf) override;
 

src/osd/OSD.cc

@@ -1934,6 +1934,8 @@ bool OSD::asok_command(string command, cmdmap_t& cmdmap, string format,
     f->dump_bool("success", success);
     f->dump_int("value", value);
     f->close_section();
+  } else if (command == "dump_objectstore_kv_stats") {
+    store->get_db_statistics(f);
   } else {
     assert(0 == "broken asok registration");
   }
@@ -2393,6 +2395,9 @@ void OSD::final_init()
 				     "get malloc extension heap property");
   assert(r == 0);
 
+  r = admin_socket->register_command("dump_objectstore_kv_stats", "dump_objectstore_kv_stats", asok_hook,
+					 "print statistics of kvdb which used by bluestore");
+  assert(r == 0);
 
   test_ops_hook = new TestOpsSocketHook(&(this->service), this->store);
   // Note: pools are CephString instead of CephPoolname because
@@ -2714,6 +2719,7 @@ int OSD::shutdown()
   cct->get_admin_socket()->unregister_command("get_latest_osdmap");
   cct->get_admin_socket()->unregister_command("set_heap_property");
   cct->get_admin_socket()->unregister_command("get_heap_property");
+  cct->get_admin_socket()->unregister_command("dump_objectstore_kv_stats");
   delete asok_hook;
   asok_hook = NULL;