pegasus_server: improve rocksdb options under PREFER_WRITE usage scenario to...

pegasus_server: improve rocksdb options under PREFER_WRITE usage scenario to avoid flush and compaction peek (#62)

README.md

+[![Build Status](https://travis-ci.org/XiaoMi/pegasus.svg?branch=master)](https://travis-ci.org/XiaoMi/pegasus)
+
 ![docs/media-img/pegasus-logo.png](docs/media-img/pegasus-logo.png)
 
 [**中文Wiki**](https://github.com/xiaomi/pegasus/wiki), [**slack channel**](https://join.slack.com/t/pegasus-kv/shared_invite/enQtMjcyMjQzOTk4Njk1LWVkMjlkMGE5Mzg1Y2M3MDc0NGYyYzQ5YzYyMGE0ZjlhMDMyNjU1ZGViYzdjZmUwNjVmNGE0ZDdkMWJiN2Q1MDY)

rdsn @ f2fadaf1

-Subproject commit 9f5c4144a8f64d875f734b6f38f764855a4e941d
+Subproject commit f2fadaf1f4414584e96e51e2c3977e2db9b4e41b

scripts/update_qt_config.sh

@@ -52,8 +52,8 @@ echo "rdsn/src/dist/block_service/local" >>$INCLUDES_OUT
 echo "rdsn/src" >> $INCLUDES_OUT
 echo "rocksdb" >>$INCLUDES_OUT
 echo "rocksdb/include" >>$INCLUDES_OUT
+echo "src" >>$INCLUDES_OUT
 echo "src/include" >>$INCLUDES_OUT
-echo "src/base" >>$INCLUDES_OUT
 echo "src/redis_protocol/proxy_lib" >>$INCLUDES_OUT
 
 # files

src/server/pegasus_event_listener.cpp

@@ -2,10 +2,6 @@
 // This source code is licensed under the Apache License Version 2.0, which
 // can be found in the LICENSE file in the root directory of this source tree.
 
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
 #include "pegasus_event_listener.h"
 
 namespace pegasus {

src/server/pegasus_server_impl.cpp

@@ -48,7 +48,6 @@ static bool chkpt_init_from_dir(const char *name, int64_t &decree)
 
 pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
     : dsn::apps::rrdb_service(r),
-      _usage_scenario(ROCKSDB_ENV_USAGE_SCENARIO_NORMAL),
       _db(nullptr),
       _is_open(false),
       _value_schema_version(0),
@@ -1483,12 +1482,16 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
             return ::dsn::ERR_LOCAL_APP_FAILURE;
         }
 
-        _manual_compact_svc.init_last_finish_time_ms(_db->GetLastManualCompactFinishTime());
-
         // only enable filter after correct value_schema_version set
         _key_ttl_compaction_filter.SetValueSchemaVersion(_value_schema_version);
         _key_ttl_compaction_filter.EnableFilter();
 
+        // update LastManualCompactFinishTime
+        _manual_compact_svc.init_last_finish_time_ms(_db->GetLastManualCompactFinishTime());
+
+        // set default usage scenario
+        set_usage_scenario(ROCKSDB_ENV_USAGE_SCENARIO_NORMAL);
+
         update_app_envs(envs);
 
         parse_checkpoints();
@@ -2318,12 +2321,34 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario)
             new_options["max_write_buffer_number"] =
                 boost::lexical_cast<std::string>(_db_opts.max_write_buffer_number);
         }
+
         if (usage_scenario == ROCKSDB_ENV_USAGE_SCENARIO_NORMAL) {
+            //
+            // write_buffer_size = random_nearby(db_opts.write_buffer_size)
+            //
+            new_options["write_buffer_size"] =
+                boost::lexical_cast<std::string>(get_random_nearby(_db_opts.write_buffer_size));
+
+            //
+            // level0_file_num_compaction_trigger = db_opts.level0_file_num_compaction_trigger
+            //
             new_options["level0_file_num_compaction_trigger"] =
                 boost::lexical_cast<std::string>(_db_opts.level0_file_num_compaction_trigger);
-        } else {
+        } else { // ROCKSDB_ENV_USAGE_SCENARIO_PREFER_WRITE
+            //
+            // write_buffer_size = random_nearby(db_opts.write_buffer_size)
+            //
+            uint64_t buffer_size =
+                dsn_random64(_db_opts.write_buffer_size, _db_opts.write_buffer_size * 2);
+            new_options["write_buffer_size"] = boost::lexical_cast<std::string>(buffer_size);
+
+            //
+            // level0_file_num_compaction_trigger =
+            //     random_nearby(db_opts.max_bytes_for_level_base) / write_buffer_size
+            //
+            uint64_t max_size = get_random_nearby(_db_opts.max_bytes_for_level_base);
             new_options["level0_file_num_compaction_trigger"] =
-                boost::lexical_cast<std::string>(_db_opts.level0_file_num_compaction_trigger * 2);
+                boost::lexical_cast<std::string>(std::max(4UL, max_size / buffer_size));
         }
     } else if (usage_scenario == ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD) {
         // refer to Options::PrepareForBulkLoad()
@@ -2335,8 +2360,8 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario)
         new_options["disable_auto_compactions"] = "true";
         new_options["max_compaction_bytes"] =
             boost::lexical_cast<std::string>(static_cast<uint64_t>(1) << 60);
-        new_options["write_buffer_size"] = boost::lexical_cast<std::string>(
-            std::max(_db_opts.write_buffer_size, (size_t)(256 * 1024 * 1024)));
+        new_options["write_buffer_size"] =
+            boost::lexical_cast<std::string>(get_random_nearby(_db_opts.write_buffer_size * 4));
         new_options["max_write_buffer_number"] =
             boost::lexical_cast<std::string>(std::max(_db_opts.max_write_buffer_number, 6));
     } else {

src/server/pegasus_server_impl.h

@@ -224,6 +224,13 @@ private:
     // return true if successfully set
     bool set_options(const std::unordered_map<std::string, std::string> &new_options);
 
+    // return random value in range of [0.75,1.25] * base_value
+    inline uint64_t get_random_nearby(uint64_t base_value)
+    {
+        uint64_t gap = base_value / 4;
+        return dsn_random64(base_value - gap, base_value + gap);
+    }
+
 private:
     dsn::gpid _gpid;
     std::string _primary_address;