feat: skip block cache deallocation to make shutdown fast (#2683)

Co-authored-by: Twice <[email protected]> Co-authored-by: hulk <[email protected]>
apache · Dec 5, 2024 · d3bca42 · d3bca42
1 parent b7e8195
commit d3bca42
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 7 deletions.
diff --git a/kvrocks.conf b/kvrocks.conf
@@ -643,6 +643,13 @@ migrate-batch-size-kb 16
 # Default: 16M
 migrate-batch-rate-limit-mb 16
 
+
+# If it is set to yes, kvrocks will skip the deallocation of block cache
+# while closing the database to speed up the shutdown
+#
+# Default: no
+# skip-block-cache-deallocation-on-close no
+
 ################################ ROCKSDB #####################################
 
 # Specify the capacity of column family block cache. A larger block cache

diff --git a/src/config/config.cc b/src/config/config.cc
@@ -239,6 +239,7 @@ Config::Config() {
       {"json-storage-format", false,
        new EnumField<JsonStorageFormat>(&json_storage_format, json_storage_formats, JsonStorageFormat::JSON)},
       {"txn-context-enabled", true, new YesNoField(&txn_context_enabled, false)},
+      {"skip-block-cache-deallocation-on-close", false, new YesNoField(&skip_block_cache_deallocation_on_close, false)},
 
       /* rocksdb options */
       {"rocksdb.compression", false,

diff --git a/src/config/config.h b/src/config/config.h
@@ -175,6 +175,8 @@ struct Config {
   // Enable transactional mode in engine::Context
   bool txn_context_enabled = false;
 
+  bool skip_block_cache_deallocation_on_close = false;
+
   struct RocksDB {
     int block_size;
     bool cache_index_and_filter_blocks;

diff --git a/src/storage/storage.cc b/src/storage/storage.cc
@@ -88,6 +88,13 @@ Storage::Storage(Config *config)
 Storage::~Storage() {
   DestroyBackup();
   CloseDB();
+  TrySkipBlockCacheDeallocationOnClose();
+}
+
+void Storage::TrySkipBlockCacheDeallocationOnClose() {
+  if (config_->skip_block_cache_deallocation_on_close) {
+    shared_block_cache_->DisownData();
+  }
 }
 
 void Storage::CloseDB() {
@@ -285,18 +292,16 @@ Status Storage::Open(DBOpenMode mode) {
     }
   }
 
-  std::shared_ptr<rocksdb::Cache> shared_block_cache;
-
   if (config_->rocks_db.block_cache_type == BlockCacheType::kCacheTypeLRU) {
-    shared_block_cache = rocksdb::NewLRUCache(block_cache_size, kRocksdbLRUAutoAdjustShardBits,
-                                              kRocksdbCacheStrictCapacityLimit, kRocksdbLRUBlockCacheHighPriPoolRatio);
+    shared_block_cache_ = rocksdb::NewLRUCache(block_cache_size, kRocksdbLRUAutoAdjustShardBits,
+                                               kRocksdbCacheStrictCapacityLimit, kRocksdbLRUBlockCacheHighPriPoolRatio);
   } else {
     rocksdb::HyperClockCacheOptions hcc_cache_options(block_cache_size, kRockdbHCCAutoAdjustCharge);
-    shared_block_cache = hcc_cache_options.MakeSharedCache();
+    shared_block_cache_ = hcc_cache_options.MakeSharedCache();
   }
 
   rocksdb::BlockBasedTableOptions metadata_table_opts = InitTableOptions();
-  metadata_table_opts.block_cache = shared_block_cache;
+  metadata_table_opts.block_cache = shared_block_cache_;
   metadata_table_opts.pin_l0_filter_and_index_blocks_in_cache = true;
   metadata_table_opts.cache_index_and_filter_blocks = cache_index_and_filter_blocks;
   metadata_table_opts.cache_index_and_filter_blocks_with_high_priority = true;
@@ -313,7 +318,7 @@ Status Storage::Open(DBOpenMode mode) {
   SetBlobDB(&metadata_opts);
 
   rocksdb::BlockBasedTableOptions subkey_table_opts = InitTableOptions();
-  subkey_table_opts.block_cache = shared_block_cache;
+  subkey_table_opts.block_cache = shared_block_cache_;
   subkey_table_opts.pin_l0_filter_and_index_blocks_in_cache = true;
   subkey_table_opts.cache_index_and_filter_blocks = cache_index_and_filter_blocks;
   subkey_table_opts.cache_index_and_filter_blocks_with_high_priority = true;

diff --git a/src/storage/storage.h b/src/storage/storage.h
@@ -21,6 +21,7 @@
 #pragma once
 
 #include <event2/bufferevent.h>
+#include <rocksdb/advanced_cache.h>
 #include <rocksdb/db.h>
 #include <rocksdb/options.h>
 #include <rocksdb/table.h>
@@ -212,6 +213,7 @@ class Storage {
   void SetWriteOptions(const Config::RocksDB::WriteOptions &config);
   Status Open(DBOpenMode mode = kDBOpenModeDefault);
   void CloseDB();
+  void TrySkipBlockCacheDeallocationOnClose();
   bool IsEmptyDB();
   void EmptyDB();
   rocksdb::BlockBasedTableOptions InitTableOptions();
@@ -380,6 +382,9 @@ class Storage {
 
   rocksdb::WriteOptions default_write_opts_ = rocksdb::WriteOptions();
 
+  // rocksdb used global block cache
+  std::shared_ptr<rocksdb::Cache> shared_block_cache_;
+
   rocksdb::Status writeToDB(engine::Context &ctx, const rocksdb::WriteOptions &options, rocksdb::WriteBatch *updates);
   void recordKeyspaceStat(const rocksdb::ColumnFamilyHandle *column_family, const rocksdb::Status &s);
 };

diff --git a/tests/gocase/unit/config/config_test.go b/tests/gocase/unit/config/config_test.go
@@ -320,3 +320,24 @@ func TestGenerateConfigsMatrix(t *testing.T) {
 	require.Contains(t, configsMatrix, util.KvrocksServerConfigs{"txn-context-enabled": "no", "resp3-enabled": "yes"})
 	require.Contains(t, configsMatrix, util.KvrocksServerConfigs{"txn-context-enabled": "no", "resp3-enabled": "no"})
 }
+
+func TestGetConfigSkipBlockCacheDeallocationOnClose(t *testing.T) {
+	srv := util.StartServer(t, map[string]string{
+		"skip-block-cache-deallocation-on-close": "yes",
+	})
+	defer srv.Close()
+
+	ctx := context.Background()
+	rdb := srv.NewClient()
+	defer func() { require.NoError(t, rdb.Close()) }()
+	val := rdb.ConfigGet(ctx, "skip-block-cache-deallocation-on-close").Val()
+	require.EqualValues(t, "yes", val["skip-block-cache-deallocation-on-close"])
+
+	// default value "no"
+	srv1 := util.StartServer(t, map[string]string{})
+	defer srv1.Close()
+
+	rdb = srv1.NewClient()
+	val = rdb.ConfigGet(ctx, "skip-block-cache-deallocation-on-close").Val()
+	require.EqualValues(t, "no", val["skip-block-cache-deallocation-on-close"])
+}