file cleanup timeout config (#2250)

* write error
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* out of storage
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* clang format
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* fix ut
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* fix #1955
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* refine code
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* compact threashold
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* changelog
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* search by id for hnsw/pq/annoy
Signed-off-by: Ngroot <yihua.mo@zilliz.com>

* fix python test
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* file cleanup timeout config
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* typo
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

* changelog
Signed-off-by: Nyhmo <yihua.mo@zilliz.com>

CHANGELOG.md

@@ -18,6 +18,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#2141 Fix server start failed if wal directory exist
 -   \#2169 Fix SingleIndexTest.IVFSQHybrid unittest
 -   \#2196 Fix server start failed if wal is disabled
+-   \#2231 Use server_config to define hard-delete delay time for segment files
 
 ## Feature
 -   \#1751 Add api SearchByID

core/conf/server_config.template

@@ -68,9 +68,13 @@ db_config:
 # secondary_path       | A semicolon-separated list of secondary directories used   | Path       |                 |
 #                      | to save vector data and index data.                        |            |                 |
 #----------------------+------------------------------------------------------------+------------+-----------------+
+# file_cleanup_timeout | time gap between soft-delete and hard-delete               | Integer    | 10 (s)          |
+#                      | range [0, 3600]                                            |            |                 |
+#----------------------+------------------------------------------------------------+------------+-----------------+
 storage_config:
   primary_path: @MILVUS_DB_PATH@
   secondary_path:
+  file_cleanup_timeout: 10
 
 #----------------------+------------------------------------------------------------+------------+-----------------+
 # Metric Config        | Description                                                | Type       | Default         |

core/src/config/Config.cpp

@@ -74,6 +74,8 @@ const char* CONFIG_STORAGE_PRIMARY_PATH = "primary_path";
 const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT = "/tmp/milvus";
 const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path";
 const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = "";
+const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT = "file_cleanup_timeout";
+const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT = "10";
 const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable";
 const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false";
 const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address";
@@ -443,6 +445,7 @@ Config::ResetDefaultConfig() {
     /* storage config */
     CONFIG_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT));
     CONFIG_CHECK(SetStorageConfigSecondaryPath(CONFIG_STORAGE_SECONDARY_PATH_DEFAULT));
+    CONFIG_CHECK(SetStorageConfigFileCleanupTimeout(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT));
     CONFIG_CHECK(SetStorageConfigS3Enable(CONFIG_STORAGE_S3_ENABLE_DEFAULT));
     CONFIG_CHECK(SetStorageConfigS3Address(CONFIG_STORAGE_S3_ADDRESS_DEFAULT));
     CONFIG_CHECK(SetStorageConfigS3Port(CONFIG_STORAGE_S3_PORT_DEFAULT));
@@ -1088,6 +1091,32 @@ Config::CheckStorageConfigSecondaryPath(const std::string& value) {
     return Status::OK();
 }
 
+Status
+Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {
+    auto status = Status::OK();
+
+    if (value.empty()) {
+        return status;
+    }
+
+    if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
+        std::string msg = "Invalid file cleanup timeout: " + value +
+                          ". Possible reason: storage_config.file_cleanup_timeout is not a positive integer.";
+        return Status(SERVER_INVALID_ARGUMENT, msg);
+    } else {
+        const int64_t min = 0, max = 3600;
+        int64_t file_cleanup_timeout = std::stoll(value);
+        if (file_cleanup_timeout < min || file_cleanup_timeout > max) {
+            std::string msg = "Invalid file cleanup timeout: " + value +
+                              ". Possible reason: storage_config.file_cleanup_timeout is not in range [" +
+                              std::to_string(min) + ", " + std::to_string(max) + "].";
+            return Status(SERVER_INVALID_ARGUMENT, msg);
+        }
+    }
+
+    return Status::OK();
+}
+
 Status
 Config::CheckStorageConfigS3Enable(const std::string& value) {
     if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
@@ -1501,6 +1530,7 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>
 }
 
 #endif
+
 /* tracing config */
 Status
 Config::CheckTracingConfigJsonConfigPath(const std::string& value) {
@@ -1854,6 +1884,15 @@ Config::GetStorageConfigSecondaryPath(std::string& value) {
     return CheckStorageConfigSecondaryPath(value);
 }
 
+Status
+Config::GetStorageConfigFileCleanupTimeup(int64_t& value) {
+    std::string str =
+        GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT);
+    CONFIG_CHECK(CheckStorageConfigFileCleanupTimeout(str));
+    value = std::stoll(str);
+    return Status::OK();
+}
+
 Status
 Config::GetStorageConfigS3Enable(bool& value) {
     std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT);
@@ -2307,6 +2346,12 @@ Config::SetStorageConfigSecondaryPath(const std::string& value) {
     return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_SECONDARY_PATH, value);
 }
 
+Status
+Config::SetStorageConfigFileCleanupTimeout(const std::string& value) {
+    CONFIG_CHECK(CheckStorageConfigFileCleanupTimeout(value));
+    return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT, value);
+}
+
 Status
 Config::SetStorageConfigS3Enable(const std::string& value) {
     CONFIG_CHECK(CheckStorageConfigS3Enable(value));
@@ -2498,12 +2543,14 @@ Config::SetLogsDeleteExceeds(const std::string& value) {
 }
 
 #ifdef MILVUS_GPU_VERSION
+
 Status
 Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
     CONFIG_CHECK(CheckEngineConfigGpuSearchThreshold(value));
     CONFIG_CHECK(SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value));
     return ExecCallBacks(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
 }
+
 #endif
 
 /* gpu resource config */

core/src/config/Config.h

@@ -70,6 +70,7 @@ extern const char* CONFIG_STORAGE_PRIMARY_PATH;
 extern const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT;
 extern const char* CONFIG_STORAGE_SECONDARY_PATH;
 extern const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT;
+extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT;
 extern const char* CONFIG_STORAGE_S3_ENABLE;
 extern const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT;
 extern const char* CONFIG_STORAGE_S3_ADDRESS;
@@ -256,6 +257,8 @@ class Config {
     Status
     CheckStorageConfigSecondaryPath(const std::string& value);
     Status
+    CheckStorageConfigFileCleanupTimeout(const std::string& value);
+    Status
     CheckStorageConfigS3Enable(const std::string& value);
     Status
     CheckStorageConfigS3Address(const std::string& value);
@@ -389,6 +392,8 @@ class Config {
     Status
     GetStorageConfigSecondaryPath(std::string& value);
     Status
+    GetStorageConfigFileCleanupTimeup(int64_t& value);
+    Status
     GetStorageConfigS3Enable(bool& value);
     Status
     GetStorageConfigS3Address(std::string& value);
@@ -514,6 +519,8 @@ class Config {
     Status
     SetStorageConfigSecondaryPath(const std::string& value);
     Status
+    SetStorageConfigFileCleanupTimeout(const std::string& value);
+    Status
     SetStorageConfigS3Enable(const std::string& value);
     Status
     SetStorageConfigS3Address(const std::string& value);

core/src/db/DBImpl.cpp

@@ -2155,11 +2155,8 @@ DBImpl::BackgroundMerge(std::set<std::string> collection_ids) {
     meta_ptr_->Archive();
 
     {
-        uint64_t ttl = 10 * meta::SECOND;  // default: file will be hard-deleted few seconds after soft-deleted
-        if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
-            ttl = meta::HOUR;
-        }
-
+        uint64_t timeout = (options_.file_cleanup_timeout_ > 0) ? options_.file_cleanup_timeout_ : 10;
+        uint64_t ttl = timeout * meta::SECOND;  // default: file will be hard-deleted few seconds after soft-deleted
         meta_ptr_->CleanUpFilesWithTTL(ttl);
     }
 

core/src/db/Options.h

@@ -72,6 +72,7 @@ struct DBOptions {
     bool insert_cache_immediately_ = false;
 
     int64_t auto_flush_interval_ = 1;
+    int64_t file_cleanup_timeout_ = 10;
 
     // wal relative configurations
     bool wal_enable_ = true;

core/src/server/DBWrapper.cpp

@@ -64,6 +64,12 @@ DBWrapper::StartService() {
 
     StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_);
 
+    s = config.GetStorageConfigFileCleanupTimeup(opt.file_cleanup_timeout_);
+    if (!s.ok()) {
+        std::cerr << s.ToString() << std::endl;
+        return s;
+    }
+
     // cache config
     s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_);
     if (!s.ok()) {
@@ -71,6 +77,14 @@ DBWrapper::StartService() {
         return s;
     }
 
+    int64_t insert_buffer_size = 1 * engine::GB;
+    s = config.GetCacheConfigInsertBufferSize(insert_buffer_size);
+    if (!s.ok()) {
+        std::cerr << s.ToString() << std::endl;
+        return s;
+    }
+    opt.insert_buffer_size_ = insert_buffer_size * engine::GB;
+
     std::string mode;
     s = config.GetServerConfigDeployMode(mode);
     if (!s.ok()) {

core/src/server/delivery/request/SearchCombineRequest.cpp

@@ -384,6 +384,22 @@ SearchCombineRequest::OnExecute() {
             return status;
         }
 
+        // avoid memcpy crash, check id count = target vector count * topk
+        if (result_ids.size() != total_count * search_topk_) {
+            status = Status(DB_ERROR, "Result count doesn't match target vectors count");
+            // let all request return
+            FreeRequests(status);
+            return status;
+        }
+
+        // avoid memcpy crash, check distance count = id count
+        if (result_distances.size() != result_ids.size()) {
+            status = Status(DB_ERROR, "Result distance and id count doesn't match");
+            // let all request return
+            FreeRequests(status);
+            return status;
+        }
+
         // step 5: construct result array
         offset = 0;
         for (auto& request : request_list_) {

sdk/examples/qps/src/ClientTest.cpp

@@ -174,7 +174,7 @@ ClientTest::InsertEntities(std::shared_ptr<milvus::Connection>& conn) {
         milvus_sdk::TimeRecorder rc(title);
         milvus::Status stat = conn->Insert(parameters_.collection_name_, "", entity_array, record_ids);
         if (!stat.ok()) {
-            std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
+            std::cout << "Insert function call status: " << stat.message() << std::endl;
         }
 //        std::cout << "InsertEntities function call status: " << stat.message() << std::endl;
 //        std::cout << "Returned id array count: " << record_ids.size() << std::endl;